From d2a7e931b2e3f2975efc73fb9d4302e6ae57dc2b Mon Sep 17 00:00:00 2001 From: spirizeon Date: Fri, 15 May 2026 04:06:16 +0530 Subject: [PATCH] deslopping: consolidate 5+3 agents to 2 agents plus Mastra Workflow - Merged research-agent, testGeneratorAgent, integrationGeneratorAgent, e2eGeneratorAgent, executorAgent into single researchTestAgent with read/write/execute tools - Created testFixWorkflow using Mastra createWorkflow: discoverFiles, researchAndGenerate, .dountil(checkAndFix), createPR - Deleted 6 unused agent files and duplicate runTestsTool - Simplified src/index.ts from 575 to 40 lines - Consolidated webhook-server to single /webhook/generate-and-test endpoint - Updated all docs and tests --- CHANGELOG-deslopping.md | 37 + docs/architecture/agents.md | 147 +--- docs/architecture/control-flow.md | 133 ++-- docs/architecture/overview.md | 68 +- docs/architecture/state-management.md | 39 +- docs/guide/how-it-works.md | 97 +-- docs/index.md | 19 +- docs/reference/agents.md | 92 +-- docs/reference/entry-points.md | 46 +- src/index.ts | 581 +-------------- src/mastra/agents/e2eGeneratorAgent.ts | 55 -- src/mastra/agents/executorAgent.ts | 26 - .../agents/integrationGeneratorAgent.ts | 49 -- src/mastra/agents/myAgent.ts | 8 - src/mastra/agents/orchestratorAgent.ts | 45 -- src/mastra/agents/research-agent.ts | 67 +- src/mastra/agents/testGeneratorAgent.ts | 31 - src/mastra/index.ts | 9 +- src/mastra/tools/fs/runTestsTool.ts | 22 - src/mastra/workflows/testFixWorkflow.ts | 440 +++++++++++ src/webhook-server.ts | 682 ++---------------- tests/e2e/webhook-server.test.ts | 67 +- 22 files changed, 880 insertions(+), 1880 deletions(-) create mode 100644 CHANGELOG-deslopping.md delete mode 100644 src/mastra/agents/e2eGeneratorAgent.ts delete mode 100644 src/mastra/agents/executorAgent.ts delete mode 100644 src/mastra/agents/integrationGeneratorAgent.ts delete mode 100644 src/mastra/agents/myAgent.ts delete mode 100644 src/mastra/agents/orchestratorAgent.ts delete mode 100644 src/mastra/agents/testGeneratorAgent.ts delete mode 100644 src/mastra/tools/fs/runTestsTool.ts create mode 100644 src/mastra/workflows/testFixWorkflow.ts diff --git a/CHANGELOG-deslopping.md b/CHANGELOG-deslopping.md new file mode 100644 index 0000000..e195540 --- /dev/null +++ b/CHANGELOG-deslopping.md @@ -0,0 +1,37 @@ +# Deslopping Changes — Architecture Consolidation + +## Summary +Reduced from 5 active + 3 unused agents → 2 agents + 1 Mastra Workflow. +Replaced procedural orchestration with Mastra Workflow pattern. + +## Files Deleted (7) +- `src/mastra/agents/testGeneratorAgent.ts` — merged into researchTestAgent +- `src/mastra/agents/integrationGeneratorAgent.ts` — merged +- `src/mastra/agents/e2eGeneratorAgent.ts` — merged +- `src/mastra/agents/executorAgent.ts` — merged +- `src/mastra/agents/orchestratorAgent.ts` — unused, replaced by Workflow +- `src/mastra/agents/myAgent.ts` — unused template +- `src/mastra/tools/fs/runTestsTool.ts` — duplicate of runner/runTestsTool.ts + +## Files Created (1) +- `src/mastra/workflows/testFixWorkflow.ts` — Mastra Workflow using createWorkflow, createStep, .then(), .dountil() + +## Files Modified (11) +- `src/mastra/agents/research-agent.ts` — rewritten as combined researchTestAgent (6 tools: readFile, writeFile, runTests, fetchAnalysis, storeTests, storeResults) +- `src/mastra/index.ts` — registers 2 agents + 1 workflow +- `src/index.ts` — 575→40 lines, just runs the workflow +- `src/webhook-server.ts` — 798→195 lines, consolidated to single /webhook/generate-and-test +- `tests/e2e/webhook-server.test.ts` — updated for new agent names + consolidated endpoints +- `docs/architecture/agents.md` — 2-agent architecture +- `docs/architecture/control-flow.md` — workflow-based flow +- `docs/architecture/overview.md` — updated diagram/table +- `docs/architecture/state-management.md` — agent name updates +- `docs/reference/agents.md` — researchTestAgent + editorAgent + testFixWorkflow +- `docs/reference/entry-points.md`, `docs/guide/how-it-works.md`, `docs/index.md` — agent reference updates + +## Key Changes +1. **researchTestAgent** autonomously determines test type (unit/integration/E2E) from file content +2. **testFixWorkflow** orchestrates: discoverFiles → researchAndGenerate → loop(checkAndFix via .dountil) → createPR +3. All context flows through Redis (unchanged pattern) +4. Fix loop uses Mastra's .dountil() instead of procedural for-loop +5. PR creation built into workflow's createPR step diff --git a/docs/architecture/agents.md b/docs/architecture/agents.md index 5705477..0504859 100644 --- a/docs/architecture/agents.md +++ b/docs/architecture/agents.md @@ -1,127 +1,38 @@ # Agents -lemon.test uses five specialized AI agents built on the Mastra framework. All agents use Cloudflare Workers AI with the model `@cf/meta/llama-3.3-70b-instruct-fp8-fast`. +lemon.test uses two specialized AI agents and one Mastra Workflow built on the Mastra framework. All agents use Cloudflare Workers AI with the model `@cf/meta/llama-3.3-70b-instruct-fp8-fast`. -## Generator Agents +## researchTestAgent -### testGeneratorAgent +**Purpose**: Combines research, test generation, and test execution into a single autonomous agent. Given a source file, it researches the code (via RAG analysis), generates appropriate vitest tests, runs them, and stores the results — all in one call. -**Purpose**: Generates vitest unit tests for individual source files. +The agent autonomously determines the test type (unit, integration, or E2E) based on the file's role in the codebase. Files containing route, service, or API patterns get integration/E2E tests; utility and pure logic files get unit tests. **Tools**: - `fetchAnalysisTool` — retrieves prior code analysis from Redis (RAG context) -- `readFileTool` — reads the source file content -- `writeFileTool` — saves the generated test file - `storeTestsTool` — persists test metadata to Redis - -**Output**: `src/__tests__/.test.ts` - -**Testing Coverage**: -- Happy path (normal expected behavior) -- Edge cases (empty input, nulls, boundary values) -- Error cases (exceptions, invalid input) -- Issues flagged in stored analysis - -**Source**: `src/mastra/agents/testGeneratorAgent.ts` - ---- - -### integrationGeneratorAgent - -**Purpose**: Generates vitest integration tests that verify interactions between multiple modules, services, and external dependencies. - -**Tools**: -- `fetchAnalysisTool` — retrieves prior code analysis from Redis +- `storeResultsTool` — persists test results to Redis - `readFileTool` — reads the source file content - `writeFileTool` — saves the generated test file -- `storeTestsTool` — persists test metadata to Redis - -**Output**: `tests/integration/.test.ts` - -**Testing Coverage**: -- Interactions between multiple modules/services -- API endpoints with real database connections -- Service layer interactions and data flow -- External integrations (databases, message queues, external APIs) -- Data transformation pipelines -- Authentication and authorization flows -- Error handling across module boundaries - -**Guidelines**: -- Test real interactions between components, not isolated units -- Use setup/teardown hooks for shared resources -- Mock only expensive or unreliable external services -- Use real database connections when possible -- Focus on data flow and state changes across boundaries -- Clean up test data in afterAll hooks - -**Source**: `src/mastra/agents/integrationGeneratorAgent.ts` - ---- - -### e2eGeneratorAgent - -**Purpose**: Generates vitest end-to-end tests that verify complete user flows and system behavior from the outside. - -**Tools**: -- `fetchAnalysisTool` — retrieves prior code analysis from Redis -- `readFileTool` — reads source files, especially entry points and routes -- `writeFileTool` — saves the generated test file -- `storeTestsTool` — persists test metadata to Redis - -**Output**: `tests/e2e/.test.ts` - -**Testing Coverage**: -- Full API request/response cycles with real server -- Complete user workflows (signup → login → use feature → logout) -- Multi-step processes and state transitions -- Authentication and authorization end-to-end flows -- Payment or transaction flows -- Data lifecycle (create → read → update → delete) -- Error recovery and edge case user journeys -- Cross-feature interactions - -**Guidelines**: -- Test from the perspective of an external user/client -- Make real HTTP requests to the application -- Set up test fixtures in beforeAll, clean up in afterAll -- Each test should be a complete, independent user flow -- Verify the full chain: request → processing → response → side effects -- Test both successful flows and error/failure paths - -**Source**: `src/mastra/agents/e2eGeneratorAgent.ts` - ---- - -## Execution Agents - -### executorAgent - -**Purpose**: Runs vitest on generated test files and stores pass/fail results in Redis. - -**Tools**: - `runTestsTool` — executes vitest on a specific test file -- `storeResultsTool` — persists test results to Redis -- `fetchAnalysisTool` — retrieves prior code analysis (available but not primary) -**Responsibilities**: -- Execute vitest with verbose output -- Capture pass/fail status for each test -- Collect full stdout/stderr output -- Parse individual test failures with error messages -- Store everything to Redis with iteration number +**Workflow**: +1. Fetch code analysis from Redis for RAG context +2. Read the source file +3. Determine test type based on file analysis +4. Generate and write tests to the appropriate directory +5. Run the tests with vitest +6. Store test metadata and results to Redis -**Output**: Test results stored to `test_results:*` keys in Redis +**Output**: Generated test file + Redis entries for test metadata and results -**Source**: `src/mastra/agents/executorAgent.ts` +**Source**: `src/mastra/agents/researchTestAgent.ts` --- -## Editor Agents - -### editorAgent +## editorAgent -**Purpose**: Reads failing test results from Redis and applies targeted code fixes to make tests pass. +**Purpose**: Reads failing test results from Redis and applies targeted code fixes to make tests pass. Unchanged from the original architecture. **Tools**: - `fetchResultsTool` — retrieves test results from Redis @@ -147,28 +58,18 @@ lemon.test uses five specialized AI agents built on the Mastra framework. All ag --- -## Unused Agents - -### orchestratorAgent +## testFixWorkflow (Mastra Workflow) -A supervisor agent that was designed to coordinate the full loop with LibSQL memory. Currently unused — the orchestration logic lives in `src/index.ts` instead. +**Purpose**: Orchestrates the full test generation → fix loop using a Mastra Workflow. Replaces the previous manual orchestration in `src/index.ts`. -**Source**: `src/mastra/agents/orchestratorAgent.ts` +**Steps**: -### research-agent +1. **discoverFiles** (procedural step) — Scans the target repository for source files. Excludes `node_modules`, `__tests__`, `.d.ts`, `seeds/`, `migrations/`, `public/`. Passes the file list to the next step. -A standalone research agent using OpenAI GPT-5-mini. Not part of the testing pipeline. +2. **researchAndGenerate** (calls researchTestAgent per file) — For each discovered file, calls `researchTestAgent` which handles research → test generation → test execution → storage to Redis in a single autonomous call. -**Source**: `src/mastra/agents/research-agent.ts` - ---- +3. **loop(checkAndFix)** — Reads test results from Redis. If failures exist, calls `editorAgent` to fix the source code, then retests by calling `researchTestAgent` again. Loops until all tests pass or a maximum of 5 iterations is reached. -## Agent Tool Matrix +4. **createPR** (procedural step) — If all tests pass and changes were made, creates a GitHub branch, commits the changes, pushes, and opens a pull request using the GitHub API. -| Agent | fetchAnalysis | readFile | writeFile | runTests | storeResults | storeTests | fetchResults | listFiles | -|---|---|---|---|---|---|---|---|---| -| testGeneratorAgent | ✅ | ✅ | ✅ | | | ✅ | | | -| integrationGeneratorAgent | ✅ | ✅ | ✅ | | | ✅ | | | -| e2eGeneratorAgent | ✅ | ✅ | ✅ | | | ✅ | | | -| executorAgent | ✅ | | | ✅ | ✅ | | | | -| editorAgent | ✅ | ✅ | ✅ | | | | ✅ | ✅ | +**Source**: `src/mastra/workflows/testFixWorkflow.ts` diff --git a/docs/architecture/control-flow.md b/docs/architecture/control-flow.md index b1a653d..0d8dccf 100644 --- a/docs/architecture/control-flow.md +++ b/docs/architecture/control-flow.md @@ -8,118 +8,84 @@ This document describes the step-by-step execution flow of lemon.test from start Push → CircleCI → Machine Runner → lemon.test → Results → CircleCI ``` -Within lemon.test, the flow is: +Within lemon.test, the flow is orchestrated by a single Mastra Workflow: ``` -Discovery → Generation → Execution → Fixing → (repeat) → Report +testFixWorkflow: + Step 1: discoverFiles (procedural) + Step 2: researchAndGenerate (researchTestAgent per file) + Step 3: loop(checkAndFix) — check results, fix, retest (max 5 iterations) + Step 4: createPR (procedural) ``` ## Detailed Execution Flow -### Phase 1: File Discovery +### Step 1: discoverFiles -When `src/index.ts` starts, it scans the target repository for source files: +Scans the target repository for source files. The researchTestAgent autonomously determines test type per file, so discovery is simpler: -**Unit test targets**: 1. Recursively scan all `.ts`/`.js` files 2. Exclude: `node_modules`, `__tests__`, `.d.ts`, `seeds/`, `migrations/`, `public/` 3. Take the first 5 files -**Integration test targets**: -1. Recursively scan all `.ts`/`.js` files -2. Filter for files containing: `routes`, `api`, `service`, `controller`, `middleware`, `handler` -3. Exclude: `node_modules`, `__tests__`, `.d.ts` -4. Take the first 5 files - -**E2E test targets**: -1. Recursively scan all `.ts`/`.js` files -2. Filter for files containing: `app`, `server`, `index`, `routes`, `auth` -3. Exclude: `node_modules`, `__tests__`, `.d.ts` -4. Take the first 3 files +No separate filtering for unit/integration/E2E targets — the agent determines the appropriate test type from the file's content and role. -### Phase 2: Test Generation +### Step 2: researchAndGenerate -For each test type, the corresponding generator agent processes files sequentially: +For each discovered source file, the workflow calls `researchTestAgent`: -**Unit test generation**: ``` For each source file: - 1. testGeneratorAgent receives prompt with file path - 2. Agent calls fetch-analysis to get RAG context - 3. Agent calls read-file to get source code - 4. Agent writes vitest unit tests - 5. Agent calls write-file to save to src/__tests__/.test.ts - 6. Agent calls store-tests to persist metadata to Redis -``` - -**Integration test generation**: -``` -For each integration target file: - 1. integrationGeneratorAgent receives prompt with file path - 2. Agent calls fetch-analysis to get RAG context + 1. researchTestAgent receives prompt with file path + 2. Agent calls fetch-analysis to get RAG context from Redis 3. Agent calls read-file to get source code - 4. Agent writes vitest integration tests - 5. Agent calls write-file to save to tests/integration/.test.ts - 6. Agent calls store-tests to persist metadata to Redis + 4. Agent autonomously determines test type (unit/integration/E2E) + 5. Agent writes vitest tests to the appropriate directory + 6. Agent calls run-tests to execute vitest + 7. Agent calls store-tests to persist metadata to Redis + 8. Agent calls store-results to persist test results to Redis ``` -**E2E test generation**: -``` -For each E2E target file: - 1. e2eGeneratorAgent receives prompt with file path - 2. Agent calls fetch-analysis to get RAG context - 3. Agent calls read-file to get source code - 4. Agent writes vitest E2E tests - 5. Agent calls write-file to save to tests/e2e/.test.ts - 6. Agent calls store-tests to persist metadata to Redis -``` +The agent handles research, generation, execution, and storage in a single autonomous call. -### Phase 3: Test-Fix Loop +### Step 3: loop(checkAndFix) -After generation, lemon.test runs an independent test-fix loop for each test type: +After generation, the workflow enters a fix loop: ``` -runTestFixLoop(testDir, label): +loop(checkAndFix): For iteration = 1 to MAX_ITERATIONS (5): - Step A: Execute Tests - For each test file in testDir: - 1. executorAgent receives prompt with test file path - 2. Agent calls run-tests to execute vitest - 3. Agent calls store-results to persist results to Redis - - testId, filePath, passed, output, failures, iteration - - Step B: Check Results + Step A: Check Results + Read test results from Redis for the current iteration If all tests passed: - Return { status: "passed", iterations: iteration } + Exit loop — proceed to createPR If iteration == MAX_ITERATIONS: - Return { status: "max_iterations", iterations: iteration } + Exit loop — report max_iterations - Step C: Fix Failures - 1. editorAgent receives prompt with iteration number + Step B: Fix Failures + 1. Call editorAgent with iteration number 2. Agent calls fetch-results to get failing tests 3. For each failing test: a. Agent calls read-file on the source file b. Agent analyzes the failure c. Agent calls write-file with the fix + patchDescription - 4. Loop back to Step A + 4. Call researchTestAgent again to retest the fixed code + 5. Store new results to Redis + 6. Loop back to Step A ``` -### Phase 4: Reporting +### Step 4: createPR -After all three test-fix loops complete, the results are summarized: +If all tests pass and changes were made: ``` -Unit tests: {status} ({iterations} iterations) -Integration tests: {status} ({iterations} iterations) -E2E tests: {status} ({iterations} iterations) +1. Create a new branch +2. Commit all changes (tests + source fixes) +3. Push to GitHub +4. Open a pull request ``` -Possible statuses: -- `passed` — all tests passed within the iteration limit -- `max_iterations` — reached the iteration limit with remaining failures -- `no_tests` — no test files were found for this type - ## Webhook Mode Flow In webhook mode (`src/webhook-server.ts`), the flow is similar but wrapped in HTTP endpoints: @@ -130,16 +96,9 @@ In webhook mode (`src/webhook-server.ts`), the flow is similar but wrapped in HT 1. Verify webhook signature (if secret is configured) 2. Clone target repo to temp workspace 3. Set LEMON_WORKSPACE to the cloned directory -4. Run test-fix loop for unit tests -5. Run test-fix loop for integration tests -6. Run test-fix loop for E2E tests -7. If all passed and changes exist: - a. Create a new branch - b. Commit all changes - c. Push to GitHub - d. Open a PR -8. Return results to caller -9. Clean up temp workspace +4. Run testFixWorkflow +5. Return results to caller +6. Clean up temp workspace ``` ### POST /webhook/generate-tests @@ -147,25 +106,17 @@ In webhook mode (`src/webhook-server.ts`), the flow is similar but wrapped in HT ``` 1. Clone target repo 2. Set LEMON_WORKSPACE -3. Run testGeneratorAgent for each source file +3. Run researchAndGenerate step of testFixWorkflow 4. Return list of generated tests 5. Clean up workspace ``` -### POST /webhook/generate-integration-tests - -Same as above but with integrationGeneratorAgent. - -### POST /webhook/generate-e2e-tests - -Same as above but with e2eGeneratorAgent. - ### POST /webhook/run-tests ``` 1. Clone target repo 2. Set LEMON_WORKSPACE -3. Run executorAgent for each test file +3. Call researchTestAgent for each test file (runs tests only) 4. Return pass/fail results 5. Clean up workspace ``` diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 2ffdc05..00a5e61 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -12,24 +12,31 @@ lemon.test is a multi-agent AI testing platform built on the Mastra framework. I │ │ Entry Points │ │ │ │ │ │ │ │ src/index.ts src/webhook-server.ts │ │ -│ │ (direct execution) (Express server, legacy mode) │ │ +│ │ (triggers workflow) (Express server, legacy mode) │ │ │ └────────────┬───────────────────────────┬──────────────────┘ │ │ │ │ │ │ ▼ ▼ │ │ ┌────────────────────────────────────────────────────────────┐ │ -│ │ Mastra Orchestrator │ │ +│ │ testFixWorkflow (Mastra Workflow) │ │ │ │ │ │ -│ │ testGeneratorAgent integrationGeneratorAgent │ │ -│ │ e2eGeneratorAgent executorAgent editorAgent │ │ -│ └────────────┬───────────────────────────┬──────────────────┘ │ -│ │ │ │ -│ ▼ ▼ │ +│ │ discoverFiles → researchAndGenerate → loop(checkAndFix) │ │ +│ │ │ │ │ +│ │ ┌───────▼────────┐ │ │ +│ │ │ researchTest │ │ │ +│ │ │ Agent │ │ │ +│ │ └───────┬────────┘ │ │ +│ │ ┌───────▼────────┐ │ │ +│ │ │ editorAgent │ │ │ +│ │ └────────────────┘ │ │ +│ └───────────────────────────┬──────────────────────────────┘ │ +│ │ │ +│ ▼ │ │ ┌──────────────────────┐ ┌────────────────────────────────┐ │ │ │ Tools │ │ Redis │ │ │ │ │ │ │ │ │ │ File I/O: │ │ code_analysis:* │ │ -│ │ - readFileTool │ │ unit_tests:* │ │ -│ │ - writeFileTool │ │ test_results:* │ │ +│ │ - readFileTool │ │ test_results:* │ │ +│ │ - writeFileTool │ │ test_metadata:* │ │ │ │ - listFilesTool │ │ code_patches:* │ │ │ │ │ │ │ │ │ │ Runner: │ │ (shared event log) │ │ @@ -56,15 +63,21 @@ lemon.test is a multi-agent AI testing platform built on the Mastra framework. I ### AI Agents -Five specialized agents powered by Mastra, each with a distinct role: +Two specialized agents powered by Mastra, orchestrated by a Mastra Workflow: | Agent | Purpose | Model | |---|---|---| -| `testGeneratorAgent` | Generates vitest unit tests | Cloudflare Workers AI | -| `integrationGeneratorAgent` | Generates integration tests | Cloudflare Workers AI | -| `e2eGeneratorAgent` | Generates E2E tests | Cloudflare Workers AI | -| `executorAgent` | Runs tests, stores results | Cloudflare Workers AI | -| `editorAgent` | Analyzes failures, fixes code | Cloudflare Workers AI | +| `researchTestAgent` | Researches code, generates tests, runs them, stores results (autonomously determines test type) | Cloudflare Workers AI | +| `editorAgent` | Analyzes failures, fixes source code | Cloudflare Workers AI | + +### testFixWorkflow + +A Mastra Workflow that orchestrates the full pipeline: + +1. **discoverFiles** — scans repo for source files +2. **researchAndGenerate** — calls researchTestAgent per file (research → gen tests → run → store) +3. **loop(checkAndFix)** — reads Redis results, calls editorAgent on failures, retests, loops up to 5 iterations +4. **createPR** — commits changes and opens a GitHub PR ### Tools @@ -78,18 +91,18 @@ Purpose-built tools that agents use to interact with the codebase: Redis serves as the shared event log and knowledge base: -- **Code Analysis** — prior analysis used as RAG context for generators +- **Code Analysis** — prior analysis used as RAG context for researchTestAgent - **Test Metadata** — generated tests with source file mappings - **Test Results** — pass/fail status, output, and failure details per iteration - **Code Patches** — every fix applied by the editor agent with descriptions ## Execution Flow -1. **Discovery** — Scan target repo for source files matching each test type -2. **Generation** — Each generator agent reads source code + analysis, writes tests -3. **Execution** — executorAgent runs vitest, stores results in Redis -4. **Fixing** — editorAgent reads failures, applies minimal source code fixes -5. **Iteration** — Steps 3-4 repeat until all pass or max iterations reached +1. **Discovery** — Scan target repo for source files +2. **Research & Generate** — researchTestAgent reads source + analysis, writes tests, runs them +3. **Fix Loop** — editorAgent reads failures, applies source code fixes, retests +4. **Iteration** — Step 3 repeats until all pass or max iterations reached +5. **PR Creation** — If all pass, commit and open a pull request ## Key Design Decisions @@ -102,14 +115,15 @@ Agents communicate through Redis rather than calling each other directly because - **Replayability** — the entire session can be replayed from Redis data - **Observability** — external tools can monitor agent behavior in real-time -### Why Specialized Agents Over One Generalist +### Why Two Agents + Workflow Instead of Five Specialized Agents -Five specialized agents produce better results than one general-purpose agent: +The 2-agent + workflow architecture was chosen over the previous 5-agent design because: -- Each agent has focused instructions tailored to its specific task -- Tool sets are minimized to only what each agent needs -- Prompts can be optimized independently for each role -- Failures are easier to diagnose and fix +- **Reduced complexity** — one agent handles the entire research→generate→execute cycle autonomously +- **Fewer round-trips** — no need to coordinate between separate generator and executor agents +- **Autonomous test typing** — the agent determines test type from file content, eliminating separate discovery pipelines +- **Workflow orchestration** — Mastra Workflows provide built-in step sequencing, looping, and error handling, replacing manual orchestration code +- **Easier debugging** — fewer agents means fewer failure points and simpler traceability ### Why vitest diff --git a/docs/architecture/state-management.md b/docs/architecture/state-management.md index 08792fb..58bbd5f 100644 --- a/docs/architecture/state-management.md +++ b/docs/architecture/state-management.md @@ -19,11 +19,11 @@ REDIS_PORT=6379 ### Code Analysis: `code_analysis:*` -**Purpose**: Prior code analysis used as RAG (Retrieval-Augmented Generation) context by generator agents. +**Purpose**: Prior code analysis used as RAG (Retrieval-Augmented Generation) context by researchTestAgent. **Written by**: External analysis process (not part of the core lemon.test loop) -**Read by**: All generator agents via `fetchAnalysisTool` +**Read by**: researchTestAgent via `fetchAnalysisTool` **Structure**: ```json @@ -35,15 +35,15 @@ REDIS_PORT=6379 } ``` -**Usage**: Generator agents fetch this before reading source files to understand the file's purpose and known issues, enabling them to write more targeted tests. +**Usage**: researchTestAgent fetches this before reading source files to understand the file's purpose and known issues, enabling it to write more targeted tests. --- -### Unit Tests: `unit_tests:*` +### Test Metadata: `test_metadata:*` -**Purpose**: Metadata about generated unit tests. +**Purpose**: Metadata about generated tests. -**Written by**: Generator agents via `storeTestsTool` +**Written by**: researchTestAgent via `storeTestsTool` **Structure**: ```json @@ -52,6 +52,7 @@ REDIS_PORT=6379 "filePath": "src/services/auth.ts", "testFilePath": "src/__tests__/auth.test.ts", "testCode": "import { describe, it, expect } from 'vitest'...", + "testType": "unit", "generatedAt": "2024-01-15T10:30:00.000Z", "status": "pending" } @@ -61,9 +62,9 @@ REDIS_PORT=6379 ### Test Results: `test_results:*` -**Purpose**: Test execution results — the primary communication channel between executorAgent and editorAgent. +**Purpose**: Test execution results — the primary communication channel between researchTestAgent and editorAgent. -**Written by**: executorAgent via `storeResultsTool` +**Written by**: researchTestAgent via `storeResultsTool` **Read by**: editorAgent via `fetchResultsTool` @@ -119,11 +120,11 @@ REDIS_PORT=6379 │ code_analysis:hash1 ← RAG context (pre-loaded) │ │ code_analysis:hash2 │ │ │ -│ unit_tests:uuid1 ← Generated test metadata │ -│ unit_tests:uuid2 │ +│ test_metadata:uuid1 ← Generated test metadata │ +│ test_metadata:uuid2 │ │ │ │ test_results:uuid1 ← Test execution results │ -│ test_results:uuid2 ← (executorAgent → editorAgent) │ +│ test_results:uuid2 ← (researchTestAgent → editorAgent) │ │ │ │ code_patches:uuid1 ← Applied code fixes (audit log) │ │ code_patches:uuid2 │ @@ -132,16 +133,13 @@ REDIS_PORT=6379 ## State Lifecycle -### During Generation +### During Research & Generation -1. Generator agents fetch `code_analysis:*` for context -2. Generator agents write tests to disk -3. Generator agents store metadata to `unit_tests:*` - -### During Execution - -1. executorAgent runs vitest -2. executorAgent stores results to `test_results:*` with iteration number +1. researchTestAgent fetches `code_analysis:*` for context +2. researchTestAgent writes tests to disk +3. researchTestAgent runs tests with vitest +4. researchTestAgent stores metadata to `test_metadata:*` +5. researchTestAgent stores results to `test_results:*` ### During Fixing @@ -149,6 +147,7 @@ REDIS_PORT=6379 2. editorAgent reads failing source files 3. editorAgent writes fixes to disk 4. writeFileTool automatically logs patches to `code_patches:*` +5. researchTestAgent retests and stores new results to `test_results:*` ### Between Iterations diff --git a/docs/guide/how-it-works.md b/docs/guide/how-it-works.md index 4418034..525d1a4 100644 --- a/docs/guide/how-it-works.md +++ b/docs/guide/how-it-works.md @@ -1,16 +1,16 @@ # How It Works -lemon.test uses five specialized AI agents that work together in a generate → run → fix loop to autonomously create and maintain tests for your codebase. +lemon.test uses two specialized AI agents orchestrated by a Mastra Workflow in a research → generate → run → fix loop to autonomously create and maintain tests for your codebase. -## The Agent Pipeline +## The Pipeline ``` ┌─────────────────────────────────────────────────────────────┐ │ lemon.test Pipeline │ │ │ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ Discover │───▶│ Generate │───▶│ Execute │ │ -│ │ Files │ │ Tests │ │ Tests │ │ +│ │ Discover │───▶│ Research & │───▶│ Check & │ │ +│ │ Files │ │ Generate │ │ Fix Loop │ │ │ └──────────────┘ └──────────────┘ └──────┬───────┘ │ │ │ │ │ ┌─────────────▼─────────┐ │ @@ -19,8 +19,8 @@ lemon.test uses five specialized AI agents that work together in a generate → │ Yes │ │ No │ │ │ │ │ │ ┌──────▼──┐ ┌───▼──────┐ │ -│ │ DONE │ │ Fix │ │ -│ │ ✅ │ │ Code │ │ +│ │ Create │ │ Fix │ │ +│ │ PR │ │ Source │ │ │ └─────────┘ └───┬──────┘ │ │ │ │ │ ┌────────────────┘ │ @@ -29,82 +29,61 @@ lemon.test uses five specialized AI agents that work together in a generate → └─────────────────────────────────────────────────────────────┘ ``` -## Three Test Types, Three Phases +## How It Works in Detail -### Phase 1: Unit Tests +### 1. File Discovery -**Target files**: Any `.ts`/`.js` file (excluding `node_modules`, `__tests__`, `.d.ts`, `seeds/`, `migrations/`, `public/`) +Scans the target repository for `.ts`/`.js` source files, excluding `node_modules`, `__tests__`, `.d.ts`, `seeds/`, `migrations/`, and `public/`. Takes the first 5 files. -**Agent**: `testGeneratorAgent` +Unlike the previous architecture, there is no separate filtering for unit, integration, or E2E targets. The researchTestAgent autonomously determines the appropriate test type from each file's content and role. -**Output**: `src/__tests__/.test.ts` +### 2. Research and Generate -The agent reads source code and any prior analysis stored in Redis, then writes comprehensive vitest unit tests covering happy paths, edge cases, and error scenarios. +For each discovered file, `researchTestAgent` handles the entire cycle: -### Phase 2: Integration Tests +1. Fetches prior code analysis from Redis for context (RAG) +2. Reads the source file +3. Determines the test type (unit, integration, or E2E) based on the file content +4. Writes comprehensive vitest tests to the appropriate directory +5. Runs the tests with vitest +6. Stores both test metadata and results to Redis -**Target files**: Files containing `routes`, `api`, `service`, `controller`, `middleware`, or `handler` +### 3. The Check-and-Fix Loop -**Agent**: `integrationGeneratorAgent` - -**Output**: `tests/integration/.test.ts` - -The agent focuses on interactions between modules, API endpoints with real database connections, service layer data flows, and cross-boundary error handling. - -### Phase 3: E2E Tests - -**Target files**: Files containing `app`, `server`, `index`, `routes`, or `auth` - -**Agent**: `e2eGeneratorAgent` - -**Output**: `tests/e2e/.test.ts` - -The agent writes end-to-end tests for complete user journeys: full API request/response cycles, multi-step workflows, authentication flows, and data lifecycle operations. - -## The Test-Fix Loop - -After generating tests for all three types, lemon.test runs an iterative fix loop for each: +After all files have been processed, the workflow enters a fix loop: ``` Iteration 1: - executorAgent runs vitest on all test files - Results stored in Redis (pass/fail, output, failures) + researchTestAgent ran tests and stored results in Redis - If all pass → DONE + If results all pass → proceed to createPR If any fail → editorAgent analyzes failures and fixes source code - + researchTestAgent retests the fixed code + New results stored in Redis + Iteration 2: - executorAgent runs vitest again (on the fixed code) - Results stored in Redis - - If all pass → DONE - If any fail → editorAgent applies more fixes + Check results from retest + If all pass → proceed to createPR + If any fail → editorAgent applies more fixes, researchTestAgent retests ...repeats up to MAX_ITERATIONS (5) ``` +### 4. Create PR + +If all tests pass, the workflow creates a GitHub branch, commits the changes (new tests + any source code fixes), pushes, and opens a pull request. + ## How Agents Communicate All agents communicate through **Redis** as a shared event log: | Key Pattern | Purpose | Written By | Read By | |---|---|---|---| -| `code_analysis:*` | Prior code analysis (RAG context) | External | Generator agents | -| `unit_tests:*` | Generated test metadata | Generator agents | — | -| `test_results:*` | Test execution results | executorAgent | editorAgent | +| `code_analysis:*` | Prior code analysis (RAG context) | External | researchTestAgent | +| `test_metadata:*` | Generated test metadata | researchTestAgent | — | +| `test_results:*` | Test execution results | researchTestAgent | editorAgent | | `code_patches:*` | Applied code fixes | editorAgent (via writeFileTool) | — | -## File Discovery Logic - -### Unit Test Discovery -Scans all `.ts`/`.js` files, excludes test directories and type definitions, takes the first 5. - -### Integration Test Discovery -Scans for files containing: `routes`, `api`, `service`, `controller`, `middleware`, `handler`. Takes the first 5. - -### E2E Test Discovery -Scans for files containing: `app`, `server`, `index`, `routes`, `auth`. Takes the first 3. - ## Execution Modes ### Machine Runner Mode (Recommended) @@ -121,19 +100,19 @@ Git Push → CircleCI → Machine Runner → lemon.test → Results → CircleCI ### Webhook Mode (Legacy) ``` -Git Push → CircleCI → Webhook → lemon.test Server → Clone Repo → Run Loop → Results +Git Push → CircleCI → Webhook → lemon.test Server → Clone Repo → Run Workflow → Results ``` - Express server receives CircleCI webhooks - Clones target repo into a temp workspace -- Runs the full test-fix loop +- Runs the testFixWorkflow - Can automatically open GitHub PRs with changes ## Configuration | Setting | Default | Description | |---|---|---| -| `MAX_ITERATIONS` | 5 | Maximum fix loop iterations per test type | +| `MAX_ITERATIONS` | 5 | Maximum fix loop iterations | | `TARGET_REPO` | `process.cwd()` | Path to the target repository | | `LEMON_WORKSPACE` | — | Working directory (set by webhook mode) | | `WEBHOOK_PORT` | 3456 | Port for the webhook server | diff --git a/docs/index.md b/docs/index.md index 4b8fb1f..295f3bc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ layout: home hero: name: lemon.test text: Your codebase. Zero blind spots. - tagline: An agentic AI testing platform that autonomously generates, executes, and fixes unit, integration, and E2E tests for TypeScript/JavaScript codebases. + tagline: An agentic AI testing platform that autonomously generates, executes, and fixes tests for TypeScript/JavaScript codebases. actions: - theme: brand text: Get Started @@ -19,7 +19,7 @@ hero: features: - icon: 🧠 title: AI-Powered Test Generation - details: Specialized agents read your source code and autonomously write comprehensive vitest unit, integration, and E2E tests — no manual test writing required. + details: A research agent reads your source code, autonomously determines test types, and writes comprehensive vitest unit, integration, and E2E tests — no manual test writing required. - icon: 🔁 title: Self-Healing Test Loop details: Tests run, failures are analyzed, and the editor agent applies source code fixes automatically. The loop iterates until everything passes. @@ -43,7 +43,7 @@ Get up and running in minutes: git push origin feature/my-branch ``` -GitHub Actions will automatically run the AI test-fix loop. +GitHub Actions will automatically run the AI test-fix workflow. ## How It Works @@ -64,16 +64,14 @@ GitHub Actions will automatically run the AI test-fix loop. │ │ │ │ │ ▼ ▼ │ │ ┌───────────────────────────┐ │ -│ │ AI Agents │ │ +│ │ testFixWorkflow │ │ │ │ │ │ -│ │ testGeneratorAgent │ │ -│ │ integrationGeneratorAgent│ │ -│ │ e2eGeneratorAgent │ │ -│ │ executorAgent │ │ +│ │ researchTestAgent │ │ │ │ editorAgent │ │ │ └───────────────────────────┘ │ │ │ -│ Generate → Run → Fix → Repeat │ +│ Research → Generate → Run → Fix │ +│ → Repeat → PR │ └─────────────────────────────────────┘ ``` @@ -81,7 +79,8 @@ GitHub Actions will automatically run the AI test-fix loop. | Concept | Description | |---|---| -| **Agents** | Five specialized AI agents powered by Mastra, each with a distinct role in the test lifecycle | +| **Agents** | Two specialized AI agents powered by Mastra: `researchTestAgent` (research + generate + execute) and `editorAgent` (fix source code) | +| **Workflow** | `testFixWorkflow` orchestrates the full pipeline: discover → researchAndGenerate → loop(checkAndFix) → createPR | | **Tools** | Purpose-built file I/O, Redis operations, and test runner tools that agents use to interact with your codebase | | **Runner** | GitHub Actions runner — your code is checked out, Docker Compose spins up Redis + AI agents, results determine job success/failure | diff --git a/docs/reference/agents.md b/docs/reference/agents.md index 5cd2ddc..c8b200b 100644 --- a/docs/reference/agents.md +++ b/docs/reference/agents.md @@ -1,10 +1,10 @@ # Agents API -Reference documentation for all AI agents in lemon.test. +Reference documentation for all AI agents and the Mastra Workflow in lemon.test. ## Mastra Instance -All agents are registered in a single Mastra instance: +All agents and the workflow are registered in a single Mastra instance: ```typescript // src/mastra/index.ts @@ -12,11 +12,11 @@ import { Mastra } from "@mastra/core/mastra"; export const mastra = new Mastra({ agents: { - testGeneratorAgent, - executorAgent, + researchTestAgent, editorAgent, - integrationGeneratorAgent, - e2eGeneratorAgent, + }, + workflows: { + testFixWorkflow, }, }); ``` @@ -24,76 +24,76 @@ export const mastra = new Mastra({ Access agents via: ```typescript -const agent = mastra.getAgent("testGeneratorAgent"); +const agent = mastra.getAgent("researchTestAgent"); const result = await agent.generate("your prompt"); ``` ---- - -## testGeneratorAgent - -| Property | Value | -|---|---| -| **ID** | `testGeneratorAgent` | -| **Model** | `cloudflare-workers-ai/@cf/meta/llama-3.3-70b-instruct-fp8-fast` | -| **Source** | `src/mastra/agents/testGeneratorAgent.ts` | - -**Tools**: `fetchAnalysisTool`, `readFileTool`, `writeFileTool`, `storeTestsTool` +Run workflows via: -**Instructions Summary**: Expert test engineer that reads source code + Redis analysis, writes comprehensive vitest unit tests covering happy paths, edge cases, error cases, and known issues. +```typescript +const workflow = mastra.getWorkflow("testFixWorkflow"); +const { runId, start } = await workflow.execute({ + triggerData: { repoPath: "/path/to/repo" }, +}); +``` --- -## integrationGeneratorAgent +## researchTestAgent | Property | Value | |---|---| -| **ID** | `integrationGeneratorAgent` | +| **ID** | `researchTestAgent` | | **Model** | `cloudflare-workers-ai/@cf/meta/llama-3.3-70b-instruct-fp8-fast` | -| **Source** | `src/mastra/agents/integrationGeneratorAgent.ts` | +| **Source** | `src/mastra/agents/researchTestAgent.ts` | -**Tools**: `fetchAnalysisTool`, `readFileTool`, `writeFileTool`, `storeTestsTool` +**Tools**: `fetchAnalysisTool`, `storeTestsTool`, `storeResultsTool`, `readFileTool`, `writeFileTool`, `runTestsTool` -**Instructions Summary**: Expert test engineer specializing in integration testing. Tests interactions between modules, API endpoints with real databases, service layer data flows, and cross-boundary error handling. +**Instructions Summary**: Autonomous test engineer that researches source code via RAG analysis, determines the appropriate test type (unit/integration/E2E) from file content, generates comprehensive vitest tests, runs them, and stores results to Redis — all in a single call. --- -## e2eGeneratorAgent +## editorAgent | Property | Value | |---|---| -| **ID** | `e2eGeneratorAgent` | +| **ID** | `editorAgent` | | **Model** | `cloudflare-workers-ai/@cf/meta/llama-3.3-70b-instruct-fp8-fast` | -| **Source** | `src/mastra/agents/e2eGeneratorAgent.ts` | +| **Source** | `src/mastra/agents/editorAgent.ts` | -**Tools**: `fetchAnalysisTool`, `readFileTool`, `writeFileTool`, `storeTestsTool` +**Tools**: `fetchResultsTool`, `fetchAnalysisTool`, `readFileTool`, `writeFileTool`, `listFilesTool` -**Instructions Summary**: Expert test engineer specializing in end-to-end testing. Tests complete user journeys, full API request/response cycles, multi-step workflows, and authentication flows from an external client perspective. +**Instructions Summary**: Senior code editor and debugger. Reads failing test results from Redis, analyzes failure messages, and applies minimal surgical fixes to source files only (never modifies test files). --- -## executorAgent +## testFixWorkflow | Property | Value | |---|---| -| **ID** | `executorAgent` | -| **Model** | `cloudflare-workers-ai/@cf/meta/llama-3.3-70b-instruct-fp8-fast` | -| **Source** | `src/mastra/agents/executorAgent.ts` | - -**Tools**: `runTestsTool`, `storeResultsTool`, `fetchAnalysisTool` +| **ID** | `testFixWorkflow` | +| **Source** | `src/mastra/workflows/testFixWorkflow.ts` | -**Instructions Summary**: Test execution agent that runs vitest, captures pass/fail status, full output, and individual test failures. Persists everything to Redis for the editor agent. - ---- +A Mastra Workflow that orchestrates the full test generation → fix → PR pipeline. -## editorAgent +### Steps -| Property | Value | -|---|---| -| **ID** | `editorAgent` | -| **Model** | `cloudflare-workers-ai/@cf/meta/llama-3.3-70b-instruct-fp8-fast` | -| **Source** | `src/mastra/agents/editorAgent.ts` | +| Step | Type | Description | +|---|---|---| +| `discoverFiles` | Procedural | Scans target repo for `.ts`/`.js` source files | +| `researchAndGenerate` | Agent call | Calls `researchTestAgent` per file (research → gen → run → store) | +| `loop(checkAndFix)` | Loop (max 5) | Reads Redis results, calls `editorAgent` on failures, retests | +| `createPR` | Procedural | Commits changes and opens a GitHub pull request | -**Tools**: `fetchResultsTool`, `fetchAnalysisTool`, `readFileTool`, `writeFileTool`, `listFilesTool` +### Execution -**Instructions Summary**: Senior code editor and debugger. Reads failing test results from Redis, analyzes failure messages, and applies minimal surgical fixes to source files only (never modifies test files). +```typescript +const workflow = mastra.getWorkflow("testFixWorkflow"); +const { runId, start } = await workflow.execute({ + triggerData: { + repoPath: "/path/to/repo", // required: path to target repository + githubToken: process.env.GITHUB_TOKEN, // optional: for PR creation + maxIterations: 5, // optional: default 5 + }, +}); +``` diff --git a/docs/reference/entry-points.md b/docs/reference/entry-points.md index 95842a7..0f2b819 100644 --- a/docs/reference/entry-points.md +++ b/docs/reference/entry-points.md @@ -4,7 +4,7 @@ lemon.test has three entry points, each serving a different execution mode. ## src/index.ts — Direct Execution -**Purpose**: Main entry point for the machine runner mode. Runs the full test generation, execution, and fix loop directly. +**Purpose**: Main entry point for the machine runner mode. Triggers the testFixWorkflow. **Execution**: ```bash @@ -12,10 +12,9 @@ npx tsx src/index.ts ``` **Flow**: -1. Discovers source files for unit, integration, and E2E tests -2. Generates tests using the three generator agents -3. Runs the test-fix loop for each test type (up to 5 iterations) -4. Prints a summary of results +1. Discovers source files +2. Runs testFixWorkflow (researchAndGenerate → loop(checkAndFix) → createPR) +3. Prints a summary of results **Environment Variables**: | Variable | Default | Description | @@ -25,16 +24,14 @@ npx tsx src/index.ts **Output**: Console logs with progress and a final summary: ``` 🏁 Done. - Unit tests: passed (2 iterations) - Integration tests: passed (1 iterations) - E2E tests: max_iterations (5 iterations) + Tests: passed (2 iterations) ``` --- ## src/webhook-server.ts — Webhook Server -**Purpose**: Express.js server that receives webhooks from CircleCI, clones target repos, and runs the test-fix loop. Legacy/alternative mode. +**Purpose**: Express.js server that receives webhooks from CircleCI, clones target repos, and runs the testFixWorkflow. Legacy/alternative mode. **Execution**: ```bash @@ -47,25 +44,25 @@ npx tsx src/webhook-server.ts #### GET /health -Health check. Returns status and list of available agents. +Health check. Returns status, list of available agents, and workflows. **Response**: ```json { "status": "ok", "agents": [ - "testGeneratorAgent", - "executorAgent", - "editorAgent", - "integrationGeneratorAgent", - "e2eGeneratorAgent" + "researchTestAgent", + "editorAgent" + ], + "workflows": [ + "testFixWorkflow" ] } ``` #### POST /webhook/test-and-fix -Full generate + run + fix loop for all test types. Synchronous — waits for completion. +Full generate + run + fix workflow. Synchronous — waits for completion. **Request Body**: ```json @@ -83,9 +80,8 @@ Full generate + run + fix loop for all test types. Synchronous — waits for com "repo": "owner/repo", "branch": "feature/my-branch", "commit": "abc123", - "unit": { "status": "passed", "iterations": 2, "files": 5 }, - "integration": { "status": "passed", "iterations": 1, "files": 3 }, - "e2e": { "status": "passed", "iterations": 1, "files": 2 }, + "iterations": 2, + "files": 5, "changedFiles": ["src/__tests__/auth.test.ts", "src/services/auth.ts"], "pr": "https://github.com/owner/repo/pull/42" } @@ -95,7 +91,7 @@ Full generate + run + fix loop for all test types. Synchronous — waits for com #### POST /webhook/generate-tests -Generate unit tests only. +Generate tests only (runs the researchAndGenerate step of testFixWorkflow). **Request Body**: ```json @@ -106,14 +102,6 @@ Generate unit tests only. } ``` -#### POST /webhook/generate-integration-tests - -Generate integration tests only. - -#### POST /webhook/generate-e2e-tests - -Generate E2E tests only. - #### POST /webhook/run-tests Run existing tests only (no generation or fixing). @@ -168,7 +156,7 @@ npx lemonx init /path/to/repo **What it does**: 1. Creates `.circleci/` directory if it doesn't exist -2. Writes a CircleCI config with three jobs: `ai-test-loop`, `ai-generate-tests`, `ai-run-tests` +2. Writes a CircleCI config with jobs for the test-fix workflow 3. Skips if the config already contains lemonx integration **Output**: CircleCI config that runs on a CircleCI machine runner with lemon.test pre-installed. diff --git a/src/index.ts b/src/index.ts index 0d9fc8f..076cb10 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,574 +1,53 @@ import "dotenv/config"; import { mastra } from "./mastra"; -import { readdir, readFile } from "fs/promises"; -import { join } from "path"; -import { exec } from "child_process"; -import { promisify } from "util"; -const execAsync = promisify(exec); - -const VERBOSE = process.env.VERBOSE !== "false" && process.env.VERBOSE !== "0"; -const DEBUG = process.env.DEBUG === "true" || process.env.DEBUG === "1"; -const LOG_PREFIX = "[LEMON]"; - -function logVerbose(...args: Parameters) { - if (DEBUG) { - const timestamp = new Date().toISOString(); - console.log(`${LOG_PREFIX} [${timestamp}]`, ...args); - } -} - -function logStep(step: string, details?: string) { - const timestamp = new Date().toISOString(); - console.log(`${LOG_PREFIX} [${timestamp}] 📍 STEP: ${step}`); - if (details) { - console.log(`${LOG_PREFIX} [${timestamp}] Details: ${details}`); - } -} - -function logAgent(agentName: string, action: string, prompt: string) { - const timestamp = new Date().toISOString(); - console.log(`${LOG_PREFIX} [${timestamp}] 🤖 AGENT: ${agentName} | ACTION: ${action}`); - console.log(`${LOG_PREFIX} [${timestamp}] Prompt Length: ${prompt.length} chars`); - console.log(`${LOG_PREFIX} [${timestamp}] Full Prompt:\n${prompt}\n${"-".repeat(80)}`); -} - -function logResponse(agentName: string, response: string, truncated?: boolean) { - const timestamp = new Date().toISOString(); - console.log(`${LOG_PREFIX} [${timestamp}] 📤 RESPONSE from ${agentName}:`); - console.log(response); - if (truncated) { - console.log(`${LOG_PREFIX} [${timestamp}] [Response truncated - full length: ${response.length} chars]`); - } -} - -function logTool(toolName: string, input: Record) { - const timestamp = new Date().toISOString(); - console.log(`${LOG_PREFIX} [${timestamp}] 🔧 TOOL CALL: ${toolName}`); - console.log(`${LOG_PREFIX} [${timestamp}] Input: ${JSON.stringify(input, null, 2)}`); -} - -function logTestResult(testFile: string, passed: boolean, output: string, failures?: { testName: string; error: string }[]) { - const timestamp = new Date().toISOString(); - const status = passed ? "✅ PASSED" : "❌ FAILED"; - console.log(`${LOG_PREFIX} [${timestamp}] 🧪 TEST RESULT: ${testFile} | ${status}`); - console.log(`${LOG_PREFIX} [${timestamp}] Full Output:\n${output}\n${"-".repeat(80)}`); - if (failures && failures.length > 0) { - console.log(`${LOG_PREFIX} [${timestamp}] Failures:`); - failures.forEach((f, i) => { - console.log(`${LOG_PREFIX} [${timestamp}] [${i + 1}] ${f.testName}: ${f.error}`); - }); - } -} - -const TARGET_REPO = process.env.TARGET_REPO ?? process.cwd(); -const LEMON_WORKSPACE = process.env.LEMON_WORKSPACE ?? "/workspace"; -const MAX_ITERATIONS = 5; const GITHUB_TOKEN = process.env.LEMONX ?? process.env.GITHUB_TOKEN ?? ""; const GITHUB_REPOSITORY = process.env.GITHUB_REPOSITORY ?? ""; const GITHUB_REF = process.env.GITHUB_REF ?? ""; const GITHUB_SHA = process.env.GITHUB_SHA ?? ""; -const PR_BRANCH = `lemon/test-fix-${Date.now()}`; +const TARGET_REPO = process.env.TARGET_REPO ?? process.cwd(); +const LEMON_WORKSPACE = process.env.LEMON_WORKSPACE ?? "/workspace"; function checkRequiredEnvVars() { const missing: string[] = []; - if (!process.env.CLOUDFLARE_ACCOUNT_ID) missing.push("CLOUDFLARE_ACCOUNT_ID"); if (!process.env.CLOUDFLARE_API_KEY) missing.push("CLOUDFLARE_API_KEY"); - const hasGitHubToken = process.env.LEMONX || process.env.GITHUB_TOKEN; - if (!hasGitHubToken) { - missing.push("LEMONX (GitHub PAT)"); - } - + if (!hasGitHubToken) missing.push("LEMONX (GitHub PAT)"); if (missing.length > 0) { - console.log(`\n${LOG_PREFIX} ⚠️ Missing required repository secrets:`); + console.log("\n[LEMON] ⚠️ Missing required repository secrets:"); missing.forEach(v => console.log(` - ${v}`)); - console.log(`\n${LOG_PREFIX} Add secrets at: Settings → Secrets and variables → Actions`); - console.log(` - CLOUDFLARE_ACCOUNT_ID`); - console.log(` - CLOUDFLARE_API_KEY`); - console.log(` - LEMONX (Personal Access Token with 'repo' scope)\n`); + console.log("\n Add secrets at: Settings → Secrets and variables → Actions"); + console.log(" - CLOUDFLARE_ACCOUNT_ID"); + console.log(" - CLOUDFLARE_API_KEY"); + console.log(" - LEMONX (Personal Access Token with 'repo' scope)\n"); } } checkRequiredEnvVars(); -async function getChangedFiles(repoPath: string): Promise { - try { - const { stdout } = await execAsync("git diff --name-only HEAD", { cwd: repoPath }); - return stdout.trim().split("\n").filter(Boolean); - } catch { - return []; - } -} - -async function discoverFiles(repoPath: string) { - const entries = await readdir(repoPath, { recursive: true }) as string[]; - return entries - .filter(f => - (f.endsWith(".ts") || f.endsWith(".js")) && - !f.includes("node_modules") && - !f.includes("__tests__") && - !f.includes(".d.ts") && - !f.includes("seeds/") && - !f.includes("migrations/") && - !f.includes("public/") - ) - .slice(0, 5); -} - -async function discoverIntegrationFiles(repoPath: string) { - const entries = await readdir(repoPath, { recursive: true }) as string[]; - return entries - .filter(f => - (f.endsWith(".ts") || f.endsWith(".js")) && - !f.includes("node_modules") && - !f.includes("__tests__") && - !f.includes(".d.ts") && - (f.includes("routes") || f.includes("api") || f.includes("service") || f.includes("controller") || f.includes("middleware") || f.includes("handler")) - ) - .slice(0, 5); -} - -async function discoverE2EFiles(repoPath: string) { - const entries = await readdir(repoPath, { recursive: true }) as string[]; - return entries - .filter(f => - (f.endsWith(".ts") || f.endsWith(".js")) && - !f.includes("node_modules") && - !f.includes("__tests__") && - !f.includes(".d.ts") && - (f.includes("app") || f.includes("server") || f.includes("index") || f.includes("routes") || f.includes("auth")) - ) - .slice(0, 3); -} - -async function discoverTestFiles(repoPath: string, testDir: string) { - try { - const entries = await readdir(join(repoPath, testDir), { recursive: true }) as string[]; - return entries - .filter(f => f.endsWith(".test.ts") || f.endsWith(".test.js")) - .map(f => `${testDir}/${f}`); - } catch { - return []; - } -} - -const unitGenerator = mastra.getAgent("testGeneratorAgent"); -const integrationGenerator = mastra.getAgent("integrationGeneratorAgent"); -const e2eGenerator = mastra.getAgent("e2eGeneratorAgent"); -const executor = mastra.getAgent("executorAgent"); -const editor = mastra.getAgent("editorAgent"); - -const generatedFiles: { path: string; content: string }[] = []; - -// ── Unit Tests ────────────────────────────────────────────────── -const unitFiles = await discoverFiles(TARGET_REPO); -logStep("DISCOVERY", `Found ${unitFiles.length} source files for unit tests`); -if (DEBUG) { - unitFiles.forEach(f => logVerbose(` - ${f}`)); -} - -console.log(`\n${LOG_PREFIX} 📝 Generating unit tests...`); -for (const file of unitFiles) { - logStep("TEST_GENERATION", `Generating unit test for: ${file}`); - - const prompt = ` - Do the following steps in order: - 1. Call fetch-analysis with filePath="${file}" to get the stored analysis context. - 2. Call read-file with path="${file}" to read the source code. - 3. Write a comprehensive vitest unit test file for this source file. - 4. Call write-file with: - - path="src/__tests__/${file.replace(/^src\//, "").replace(/\.ts$/, ".test.ts")}" - - content = the full test file you wrote - 5. Call store-tests with: - - filePath="${file}" - - testFilePath="src/__tests__/${file.replace(/^src\//, "").replace(/\.ts$/, ".test.ts")}" - - testCode = the full test file content - Do all 5 steps now. - `; - - logAgent("testGeneratorAgent", "GENERATE_UNIT_TEST", prompt); - - const res = await unitGenerator.generate(prompt); - - logResponse("testGeneratorAgent", res.text, true); - - console.log(`${LOG_PREFIX} ✓ Test generated for: ${file}`); -} - -// ── Integration Tests ─────────────────────────────────────────── -const integrationFiles = await discoverIntegrationFiles(TARGET_REPO); -logStep("DISCOVERY", `Found ${integrationFiles.length} files for integration tests`); -if (DEBUG) { - integrationFiles.forEach(f => logVerbose(` - ${f}`)); -} - -if (integrationFiles.length > 0) { - console.log(`\n${LOG_PREFIX} 📝 Generating integration tests...`); - for (const file of integrationFiles) { - logStep("TEST_GENERATION", `Generating integration test for: ${file}`); - - const prompt = ` - Do the following steps in order: - 1. Call fetch-analysis with filePath="${file}" to get the stored analysis context. - 2. Call read-file with path="${file}" to read the source code. - 3. Write a comprehensive vitest integration test file for this source file. - 4. Call write-file with: - - path="tests/integration/${file.replace(/^src\//, "").replace(/\.ts$/, ".test.ts")}" - - content = the full test file you wrote - 5. Call store-tests with: - - filePath="${file}" - - testFilePath="tests/integration/${file.replace(/^src\//, "").replace(/\.ts$/, ".test.ts")}" - - testCode = the full test file content - Do all 5 steps now. - `; - - logAgent("integrationGeneratorAgent", "GENERATE_INTEGRATION_TEST", prompt); - - const res = await integrationGenerator.generate(prompt); - - logResponse("integrationGeneratorAgent", res.text, true); - - console.log(`${LOG_PREFIX} ✓ Integration test generated for: ${file}`); - } -} - -// ── E2E Tests ─────────────────────────────────────────────────── -const e2eFiles = await discoverE2EFiles(TARGET_REPO); -logStep("DISCOVERY", `Found ${e2eFiles.length} files for E2E tests`); -if (DEBUG) { - e2eFiles.forEach(f => logVerbose(` - ${f}`)); -} - -if (e2eFiles.length > 0) { - console.log(`\n${LOG_PREFIX} 📝 Generating E2E tests...`); - for (const file of e2eFiles) { - logStep("TEST_GENERATION", `Generating E2E test for: ${file}`); - - const prompt = ` - Do the following steps in order: - 1. Call fetch-analysis with filePath="${file}" to get the stored analysis context. - 2. Call read-file with path="${file}" to read the source code. - 3. Write a comprehensive vitest E2E test file for this source file. - 4. Call write-file with: - - path="tests/e2e/${file.replace(/^src\//, "").replace(/\.ts$/, ".test.ts")}" - - content = the full test file you wrote - 5. Call store-tests with: - - filePath="${file}" - - testFilePath="tests/e2e/${file.replace(/^src\//, "").replace(/\.ts$/, ".test.ts")}" - - testCode = the full test file content - Do all 5 steps now. - `; - - logAgent("e2eGeneratorAgent", "GENERATE_E2E_TEST", prompt); - - const res = await e2eGenerator.generate(prompt); - - logResponse("e2eGeneratorAgent", res.text, true); - - console.log(`${LOG_PREFIX} ✓ E2E test generated for: ${file}`); - } -} - -// ── Run + fix loop for all test types ─────────────────────────── -async function runTestFixLoop(testDir: string, label: string) { - logStep("TEST_FIX_LOOP_START", `${label} tests | max iterations: ${MAX_ITERATIONS}`); - - for (let iteration = 1; iteration <= MAX_ITERATIONS; iteration++) { - logStep(`ITERATION_${iteration}`, `Running ${label} tests...`); - - const testFiles = await discoverTestFiles(TARGET_REPO, testDir); - if (testFiles.length === 0) { - logStep("DISCOVERY", `No ${label} test files found. Skipping.`); - return { status: "no_tests", iterations: 0, changedFiles: [] }; - } - - logVerbose(`Discovered ${testFiles.length} test files:`, testFiles); - - let allPassed = true; - for (const testFile of testFiles) { - console.log(`${LOG_PREFIX} 🧪 Executing: ${testFile}`); - - const prompt = ` - Do the following steps in order: - 1. Call run-tests with testFilePath="${testFile}" - 2. Call store-results with: - - testId = any unique string - - filePath = "${testFile}" - - passed = true or false based on run-tests result - - output = the full output from run-tests - - failures = array of {testName, error} objects from the run-tests result - - iteration = ${iteration} - Do both steps now. - `; - - logAgent("executorAgent", "RUN_TESTS", prompt); - - const res = await executor.generate(prompt); - - const passed = !res.text.toLowerCase().includes("fail") && !res.text.toLowerCase().includes("error"); - logTestResult(testFile, passed, res.text); - - if (!passed) { - allPassed = false; - } - } - - if (allPassed) { - logStep("ALL_TESTS_PASSED", `All ${label} tests passed on iteration ${iteration}!`); - return { status: "passed", iterations: iteration, changedFiles: await getChangedFiles(TARGET_REPO) }; - } - - if (iteration === MAX_ITERATIONS) { - logStep("MAX_ITERATIONS_REACHED", `Max iterations reached for ${label} tests.`); - return { status: "max_iterations", iterations: iteration, changedFiles: await getChangedFiles(TARGET_REPO) }; - } - - logStep(`FIXING_ITERATION_${iteration}`, `Fixing ${label} failures...`); - - const fixPrompt = ` - Do the following steps in order: - 1. Call fetch-results with iteration=${iteration} to get failing tests. - 2. For each failing test, call read-file on the source file being tested. - 3. For each failing test, fix the source file and call write-file to save it with: - - patchDescription = a short description of what you fixed - - iteration = ${iteration} - Do all steps now. - `; - - logAgent("editorAgent", "APPLY_FIXES", fixPrompt); - - const results = await editor.generate(fixPrompt); - - logResponse("editorAgent", results.text, true); - - logVerbose(`Editor applied fixes for iteration ${iteration}`); - } - - return { status: "completed", iterations: MAX_ITERATIONS, changedFiles: await getChangedFiles(TARGET_REPO) }; -} - -const unitResult = await runTestFixLoop("src/__tests__", "unit"); -const integrationResult = await runTestFixLoop("tests/integration", "integration"); -const e2eResult = await runTestFixLoop("tests/e2e", "E2E"); - -console.log("\n🏁 Done."); -console.log(` Unit tests: ${unitResult.status} (${unitResult.iterations} iterations)`); -console.log(` Integration tests: ${integrationResult.status} (${integrationResult.iterations} iterations)`); -console.log(` E2E tests: ${e2eResult.status} (${e2eResult.iterations} iterations)`); - -// ── GitHub API Helper Functions ────────────────────────────────────── -async function getBaseBranchSha(owner: string, repo: string, branch: string): Promise { - try { - const res = await fetch( - `https://api.github.com/repos/${owner}/${repo}/branches/${branch}`, - { - headers: { - Authorization: `Bearer ${GITHUB_TOKEN}`, - Accept: "application/vnd.github+json", - }, - } - ); - if (!res.ok) return null; - const data = await res.json(); - return data.commit.sha; - } catch { - return null; - } -} - -async function createBranch(owner: string, repo: string, branch: string, baseSha: string): Promise { - const res = await fetch( - `https://api.github.com/repos/${owner}/${repo}/git/refs`, - { - method: "POST", - headers: { - Authorization: `Bearer ${GITHUB_TOKEN}`, - Accept: "application/vnd.github+json", - "Content-Type": "application/json", - }, - body: JSON.stringify({ - ref: `refs/heads/${branch}`, - sha: baseSha, - }), - } - ); - - if (!res.ok) { - const err = await res.text(); - throw new Error(`Failed to create branch ${branch}: ${err}`); - } -} - -async function getFileSha(owner: string, repo: string, path: string, branch: string): Promise { - try { - const res = await fetch( - `https://api.github.com/repos/${owner}/${repo}/contents/${path}?ref=${branch}`, - { - headers: { - Authorization: `Bearer ${GITHUB_TOKEN}`, - Accept: "application/vnd.github+json", - }, - } - ); - if (!res.ok) return null; - const data = await res.json(); - return data.sha; - } catch { - return null; - } -} - -async function uploadFileToGitHub( - owner: string, - repo: string, - filePath: string, - content: string, - branch: string -): Promise { - const sha = await getFileSha(owner, repo, filePath, branch); - - const res = await fetch( - `https://api.github.com/repos/${owner}/${repo}/contents/${filePath}`, - { - method: "PUT", - headers: { - Authorization: `Bearer ${GITHUB_TOKEN}`, - Accept: "application/vnd.github+json", - "Content-Type": "application/json", - }, - body: JSON.stringify({ - message: `🍋 lemon: generated ${filePath}`, - content: Buffer.from(content).toString("base64"), - branch, - sha: sha ?? undefined, - }), - } - ); - - if (!res.ok) { - const err = await res.text(); - throw new Error(`Failed to upload ${filePath}: ${err}`); - } -} - -async function collectGeneratedFiles(): Promise<{ path: string; content: string }[]> { - const files: { path: string; content: string }[] = []; - const dirs = ["src/__tests__", "tests/integration", "tests/e2e"]; - - for (const dir of dirs) { - try { - const entries = await readdir(join(TARGET_REPO, dir), { recursive: true }) as string[]; - for (const entry of entries) { - if (entry.endsWith(".test.ts") || entry.endsWith(".test.js")) { - const fullPath = join(TARGET_REPO, dir, entry); - const content = await readFile(fullPath, "utf-8"); - files.push({ path: `${dir}/${entry}`, content }); - } - } - } catch { - // Directory doesn't exist, skip - } - } - - return files; -} - -// ── PR Creation ────────────────────────────────────────────────── -async function createPR(): Promise { - if (!GITHUB_TOKEN || !GITHUB_REPOSITORY) { - console.log(" ⚠️ GITHUB_TOKEN or GITHUB_REPOSITORY not set — skipping PR creation"); - return null; - } - - const [owner, repo] = GITHUB_REPOSITORY.split("/"); - const baseBranch = GITHUB_REF.replace("refs/heads/", "").replace("refs/pull/", "").replace("/merge", ""); - - const prTitle = `🍋 lemon: auto-generated tests + fixes for ${baseBranch}`; - const changedFiles = [ - ...(unitResult.changedFiles || []), - ...(integrationResult.changedFiles || []), - ...(e2eResult.changedFiles || []), - ]; - - const prBody = `## 🍋 lemon — AI Test Report - -**Branch:** ${baseBranch} -**Commit:** ${GITHUB_SHA.slice(0, 7)} - -### Test Results -| Test Type | Status | Iterations | -|---|---|---| -| Unit | ${unitResult.status} | ${unitResult.iterations} | -| Integration | ${integrationResult.status} | ${integrationResult.iterations} | -| E2E | ${e2eResult.status} | ${e2eResult.iterations} | - -### What changed -- Generated vitest unit, integration, and E2E tests for source files -- Ran tests and collected pass/fail results -- Applied code fixes to make tests pass - -### Changed files -${changedFiles.length > 0 ? changedFiles.map((f: string) => `- \`${f}\``).join("\n") : "No files changed"} -`; - - try { - const files = await collectGeneratedFiles(); - if (files.length === 0) { - console.log(" ℹ️ No generated test files found — skipping PR creation"); - return null; - } - - const baseSha = await getBaseBranchSha(owner, repo, baseBranch); - if (!baseSha) { - console.log(" ❌ Could not get base branch SHA"); - return null; - } - - await createBranch(owner, repo, PR_BRANCH, baseSha); - console.log(` 🌿 Created branch: ${PR_BRANCH}`); - - console.log(` 📤 Uploading ${files.length} files to GitHub...`); - for (const file of files) { - await uploadFileToGitHub(owner, repo, file.path, file.content, PR_BRANCH); - console.log(` ✓ ${file.path}`); - } - - const res = await fetch(`https://api.github.com/repos/${owner}/${repo}/pulls`, { - method: "POST", - headers: { - Authorization: `Bearer ${GITHUB_TOKEN}`, - Accept: "application/vnd.github+json", - "Content-Type": "application/json", - "X-GitHub-Api-Version": "2022-11-28", - }, - body: JSON.stringify({ - title: prTitle, - body: prBody, - head: PR_BRANCH, - base: baseBranch, - }), - }); - - if (!res.ok) { - const err = await res.text(); - console.log(` ❌ Failed to open PR: ${err}`); - return null; - } - - const data = await res.json(); - console.log(` ✅ PR created: ${data.html_url}`); - return data.html_url; - } catch (err: any) { - console.log(` ❌ PR creation failed: ${err.message}`); - return null; - } -} - -const prUrl = await createPR(); -if (prUrl) { - console.log(`\n🎉 Pull request opened: ${prUrl}`); +console.log("\n[LEMON] 🚀 Starting test-fix workflow..."); +console.log(` Target: ${TARGET_REPO}`); +console.log(` Workspace: ${LEMON_WORKSPACE}`); + +const workflow = mastra.getWorkflow("testFixWorkflow"); +const result = await workflow.start({ + inputData: { + repoPath: TARGET_REPO, + githubToken: GITHUB_TOKEN, + githubRepo: GITHUB_REPOSITORY, + githubRef: GITHUB_REF, + githubSha: GITHUB_SHA, + }, +}); + +const output = result.output as { prUrl: string | null } | undefined; + +console.log("\n[LEMON] 🏁 Workflow complete."); +if (output?.prUrl) { + console.log(` 🎉 PR: ${output.prUrl}`); +} else { + console.log(" No PR created (missing token, no changes, or workflow incomplete)"); } process.exit(0); diff --git a/src/mastra/agents/e2eGeneratorAgent.ts b/src/mastra/agents/e2eGeneratorAgent.ts deleted file mode 100644 index 74e0cb3..0000000 --- a/src/mastra/agents/e2eGeneratorAgent.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { Agent } from "@mastra/core/agent"; -import { fetchAnalysisTool } from "../tools/redis/fetchAnalysisTool"; -import { storeTestsTool } from "../tools/redis/storeTestsTool"; -import { readFileTool } from "../tools/fs/readFileTool"; -import { writeFileTool } from "../tools/fs/writeFileTool"; - -export const e2eGeneratorAgent = new Agent({ - id: "e2eGeneratorAgent", - name: "E2E Test Generator Agent", - description: "Generates vitest end-to-end tests that verify complete user flows and system behavior from the outside", - instructions: ` - You are an expert test engineer specializing in end-to-end (E2E) testing. Your job: - - 1. Use fetch-analysis to retrieve prior code analysis from Redis — this is your RAG knowledge base. - It tells you what each file does, its language, and known issues. - 2. Use read-file to read the actual source file content, especially entry points, routes, and API handlers. - 3. Write comprehensive vitest E2E tests covering complete user journeys and system flows: - - Full API request/response cycles with real server - - Complete user workflows (signup → login → use feature → logout) - - Multi-step processes and state transitions - - Authentication and authorization end-to-end flows - - Payment or transaction flows - - Data lifecycle (create → read → update → delete) - - Error recovery and edge case user journeys - - Cross-feature interactions - 4. Use write-file to save the test to tests/e2e/.test.ts - 5. Use store-tests to persist the test metadata to Redis. - - E2E test guidelines: - - Use vitest syntax: import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' - - Test from the perspective of an external user/client — no internal knowledge - - Make real HTTP requests to the application (use supertest or fetch) - - Set up test fixtures in beforeAll, clean up in afterAll - - Each test should be a complete, independent user flow - - Verify the full chain: request → processing → response → side effects - - Test both successful flows and error/failure paths - - Include assertions on database state changes when relevant - - Test rate limiting, validation, and security boundaries - - Use realistic test data that mirrors production scenarios - - Write tests that would catch E2E bugs like: - - Broken user flows due to misconfigured routes - - Missing middleware in the request pipeline - - Incorrect error responses to clients - - State not persisting correctly across requests - - Authentication/authorization bypasses - - Data inconsistency between operations - - Missing or incorrect HTTP status codes - - Session management issues - - Focus on testing what the user experiences, not implementation details. - `, - model: "cloudflare-workers-ai/@cf/qwen/qwen3-30b-a3b-fp8", - tools: { fetchAnalysisTool, storeTestsTool, readFileTool, writeFileTool }, -}); diff --git a/src/mastra/agents/executorAgent.ts b/src/mastra/agents/executorAgent.ts deleted file mode 100644 index 903b84e..0000000 --- a/src/mastra/agents/executorAgent.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Agent } from "@mastra/core/agent"; -import { runTestsTool } from "../tools/runner/runTestsTool"; -import { storeResultsTool } from "../tools/redis/storeResultsTool"; -import { fetchAnalysisTool } from "../tools/redis/fetchAnalysisTool"; - -export const executorAgent = new Agent({ - id: "executorAgent", - name: "Test Executor Agent", - description: "Runs vitest on generated test files and stores pass/fail results in Redis", - instructions: ` - You are a test execution agent. Your job: - - 1. Use run-tests to execute the given test file via vitest. - 2. Collect pass/fail status, full output, and any individual test failures. - 3. Use store-results to persist everything to Redis including: - - Which tests passed or failed - - The full stdout output - - Specific failure messages and errors - - The current iteration number - - Be precise when capturing failure details — the editor agent depends on them. - Always store results even if all tests pass. - `, - model: "cloudflare-workers-ai/@cf/qwen/qwen3-30b-a3b-fp8", - tools: { runTestsTool, storeResultsTool, fetchAnalysisTool }, -}); diff --git a/src/mastra/agents/integrationGeneratorAgent.ts b/src/mastra/agents/integrationGeneratorAgent.ts deleted file mode 100644 index e7f9cc6..0000000 --- a/src/mastra/agents/integrationGeneratorAgent.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { Agent } from "@mastra/core/agent"; -import { fetchAnalysisTool } from "../tools/redis/fetchAnalysisTool"; -import { storeTestsTool } from "../tools/redis/storeTestsTool"; -import { readFileTool } from "../tools/fs/readFileTool"; -import { writeFileTool } from "../tools/fs/writeFileTool"; - -export const integrationGeneratorAgent = new Agent({ - id: "integrationGeneratorAgent", - name: "Integration Test Generator Agent", - description: "Generates vitest integration tests that verify interactions between multiple modules, services, and external dependencies", - instructions: ` - You are an expert test engineer specializing in integration testing. Your job: - - 1. Use fetch-analysis to retrieve prior code analysis from Redis — this is your RAG knowledge base. - It tells you what each file does, its language, and known issues. - 2. Use read-file to read the actual source file content. - 3. Write comprehensive vitest integration tests covering: - - Interactions between multiple modules/services - - API endpoints with real database connections (use test database if available) - - Service layer interactions and data flow - - External integrations (databases, message queues, external APIs) - - Data transformation pipelines - - Authentication and authorization flows - - Error handling across module boundaries - 4. Use write-file to save the test to tests/integration/.test.ts - 5. Use store-tests to persist the test metadata to Redis. - - Integration test guidelines: - - Use vitest syntax: import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' - - Test real interactions between components, not isolated units - - Use setup/teardown hooks (beforeAll/afterAll) for shared resources - - Mock only external services that are expensive or unreliable (third-party APIs) - - Use real database connections when possible (prefer test databases) - - Focus on data flow and state changes across boundaries - - Include both happy path and error scenarios - - Test edge cases in cross-module interactions - - Clean up any test data in afterAll hooks - - Add clear descriptions for what each test verifies - - Write tests that would catch integration bugs like: - - Data format mismatches between services - - Missing error handling at boundaries - - Race conditions in async operations - - Incorrect transaction handling - - Missing or incorrect middleware behavior - `, - model: "cloudflare-workers-ai/@cf/qwen/qwen3-30b-a3b-fp8", - tools: { fetchAnalysisTool, storeTestsTool, readFileTool, writeFileTool }, -}); diff --git a/src/mastra/agents/myAgent.ts b/src/mastra/agents/myAgent.ts deleted file mode 100644 index ec8d146..0000000 --- a/src/mastra/agents/myAgent.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { Agent } from "@mastra/core/agent"; - -export const myAgent = new Agent({ - id: "my-agent", - name: "My Agent", - instructions: "You are a helpful assistant", - model: "cloudflare-workers-ai/@cf/meta/llama-3.3-70b-instruct-fp8-fast", -}); diff --git a/src/mastra/agents/orchestratorAgent.ts b/src/mastra/agents/orchestratorAgent.ts deleted file mode 100644 index b0cae77..0000000 --- a/src/mastra/agents/orchestratorAgent.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { Agent } from "@mastra/core/agent"; -import { Memory } from "@mastra/memory"; -import { LibSQLStore } from "@mastra/libsql"; -import { testGeneratorAgent } from "./testGeneratorAgent"; -import { executorAgent } from "./executorAgent"; -import { editorAgent } from "./editorAgent"; -import { fetchResultsTool } from "../tools/redis/fetchResultsTool"; -import { listFilesTool } from "../tools/fs/listFilesTool"; - -export const orchestratorAgent = new Agent({ - id: "orchestratorAgent", - name: "Orchestrator", - description: "Coordinates the full test-fix-retest loop until all tests pass", - instructions: ` - You are a test orchestration supervisor. You manage three specialist agents: - - testGeneratorAgent: writes unit tests using Redis knowledge as RAG - - executorAgent: runs the tests and stores results - - editorAgent: reads failures and fixes source code - - Your loop: - 1. Call testGeneratorAgent to generate tests for the given source files. - 2. Call executorAgent to run those tests and store results. - 3. Use fetch-results to check if all tests passed. - 4. If tests failed: - a. Call editorAgent to fix the failing code. - b. Call executorAgent again with incremented iteration. - c. Repeat from step 3. - 5. Stop when all tests pass OR after 5 iterations max to avoid infinite loops. - - Track the iteration count yourself. Report a summary at the end: - - Which files were tested - - How many iterations it took - - What fixes were applied - - Final pass/fail status - `, - model: "cloudflare-workers-ai/@cf/qwen/qwen3-30b-a3b-fp8", - agents: { testGeneratorAgent, executorAgent, editorAgent }, - tools: { fetchResultsTool, listFilesTool }, - memory: new Memory({ - storage: new LibSQLStore({ - id: "orchestrator-storage", - url: "file:./orchestrator.db", - }), - }), -}); diff --git a/src/mastra/agents/research-agent.ts b/src/mastra/agents/research-agent.ts index 63ed578..5ea99ff 100644 --- a/src/mastra/agents/research-agent.ts +++ b/src/mastra/agents/research-agent.ts @@ -1,15 +1,56 @@ -import { Agent } from '@mastra/core/agent' +import { Agent } from "@mastra/core/agent"; +import { fetchAnalysisTool } from "../tools/redis/fetchAnalysisTool"; +import { storeTestsTool } from "../tools/redis/storeTestsTool"; +import { storeResultsTool } from "../tools/redis/storeResultsTool"; +import { readFileTool } from "../tools/fs/readFileTool"; +import { writeFileTool } from "../tools/fs/writeFileTool"; +import { runTestsTool } from "../tools/runner/runTestsTool"; -export const researchAgent = new Agent({ - id: 'research-agent', - name: 'Research Specialist', +export const researchTestAgent = new Agent({ + id: "researchTestAgent", + name: "Research & Test Agent", description: - 'Specializes in gathering factual information and data on any topic. ' + - 'Returns concise bullet-point summaries with key facts and sources. ' + - 'Does not write full articles or narrative content.', - instructions: - 'You are a research specialist. When given a topic, gather key facts, ' + - 'statistics, and information. Present findings as clear bullet points. ' + - 'Include sources when possible. Focus on accuracy and completeness.', - model: 'openai/gpt-5-mini', -}) + "Researches source code, generates vitest tests (unit/integration/E2E), executes them, and stores all context in Redis", + instructions: ` + You are an expert software engineer specializing in code analysis and test generation. + + For each source file, follow this pipeline: + + ## 1. RESEARCH + - Use fetch-analysis to retrieve any prior code analysis from Redis (RAG context) + - Use read-file to read the source file content + - Analyze the file's role: is it a utility/helper (unit test), a service/api handler (integration test), or an entry point/route (E2E test)? + + ## 2. GENERATE TESTS + Based on your research, determine which test type(s) to write: + - **Unit tests** — for isolated functions, utilities, pure logic → save to src/__tests__/.test.ts + - **Integration tests** — for module interactions, API handlers, services, middleware → save to tests/integration/.test.ts + - **E2E tests** — for complete user flows, entry points, auth, routes → save to tests/e2e/.test.ts + + Write comprehensive vitest tests covering: + - Happy path (normal expected behavior) + - Edge cases (empty input, nulls, boundary values) + - Error cases (exceptions, invalid input) + - Issues flagged in stored analysis + + Follow vitest conventions: import { describe, it, expect, vi, beforeAll, afterAll } from 'vitest' + Mock external dependencies with vi.mock(). + Use write-file to save each test file. + Use store-tests to persist metadata to Redis for each test file. + + ## 3. EXECUTE + - Use run-tests to execute each generated test file + - Use store-results to persist pass/fail data, output, and failure details to Redis with iteration=1 + + Be thorough and precise. Your research context and test results will be used by the editor agent to fix any failures. + `, + model: "cloudflare-workers-ai/@cf/qwen/qwen3-30b-a3b-fp8", + tools: { + fetchAnalysisTool, + storeTestsTool, + storeResultsTool, + readFileTool, + writeFileTool, + runTestsTool, + }, +}); diff --git a/src/mastra/agents/testGeneratorAgent.ts b/src/mastra/agents/testGeneratorAgent.ts deleted file mode 100644 index eee1f13..0000000 --- a/src/mastra/agents/testGeneratorAgent.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Agent } from "@mastra/core/agent"; -import { fetchAnalysisTool } from "../tools/redis/fetchAnalysisTool"; -import { storeTestsTool } from "../tools/redis/storeTestsTool"; -import { readFileTool } from "../tools/fs/readFileTool"; -import { writeFileTool } from "../tools/fs/writeFileTool"; - -export const testGeneratorAgent = new Agent({ - id: "testGeneratorAgent", - name: "Test Generator Agent", - description: "Generates vitest unit tests for source files using Redis code analysis as knowledge context", - instructions: ` - You are an expert test engineer. Your job: - - 1. Use fetch-analysis to retrieve prior code analysis from Redis — this is your RAG knowledge base. - It tells you what each file does, its language, and known issues. - 2. Use read-file to read the actual source file content. - 3. Write comprehensive vitest unit tests covering: - - Happy path (normal expected behavior) - - Edge cases (empty input, nulls, boundary values) - - Error cases (exceptions, invalid input) - - Any issues flagged in the stored analysis - 4. Use write-file to save the test to src/__tests__/.test.ts - 5. Use store-tests to persist the test metadata to Redis. - - Write tests using vitest syntax: import { describe, it, expect, vi } from 'vitest' - Mock external dependencies with vi.mock(). - Be thorough — aim for high coverage. - `, - model: "cloudflare-workers-ai/@cf/qwen/qwen3-30b-a3b-fp8", - tools: { fetchAnalysisTool, storeTestsTool, readFileTool, writeFileTool }, -}); diff --git a/src/mastra/index.ts b/src/mastra/index.ts index aca7b5a..70e1152 100644 --- a/src/mastra/index.ts +++ b/src/mastra/index.ts @@ -1,10 +1,9 @@ import { Mastra } from "@mastra/core/mastra"; -import { testGeneratorAgent } from "./agents/testGeneratorAgent"; -import { executorAgent } from "./agents/executorAgent"; +import { researchTestAgent } from "./agents/research-agent"; import { editorAgent } from "./agents/editorAgent"; -import { integrationGeneratorAgent } from "./agents/integrationGeneratorAgent"; -import { e2eGeneratorAgent } from "./agents/e2eGeneratorAgent"; +import { testFixWorkflow } from "./workflows/testFixWorkflow"; export const mastra = new Mastra({ - agents: { testGeneratorAgent, executorAgent, editorAgent, integrationGeneratorAgent, e2eGeneratorAgent }, + agents: { researchTestAgent, editorAgent }, + workflows: { testFixWorkflow }, }); diff --git a/src/mastra/tools/fs/runTestsTool.ts b/src/mastra/tools/fs/runTestsTool.ts deleted file mode 100644 index 9539929..0000000 --- a/src/mastra/tools/fs/runTestsTool.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { createTool } from "@mastra/core/tools"; -import { z } from "zod"; -import { readdir } from "fs/promises"; -import { join } from "path"; - -export const listFilesTool = createTool({ - id: "list-files", - description: "List all source files in a directory (recursively)", - inputSchema: z.object({ dir: z.string() }), - outputSchema: z.object({ files: z.array(z.string()) }), - execute: async ({ context }) => { - const entries = await readdir(join(process.cwd(), context.dir), { - recursive: true, - }) as string[]; - const files = entries.filter(f => - (f.endsWith(".ts") || f.endsWith(".js")) && - !f.includes("__tests__") && - !f.includes("node_modules") - ); - return { files }; - }, -}); diff --git a/src/mastra/workflows/testFixWorkflow.ts b/src/mastra/workflows/testFixWorkflow.ts new file mode 100644 index 0000000..2d874e8 --- /dev/null +++ b/src/mastra/workflows/testFixWorkflow.ts @@ -0,0 +1,440 @@ +import { createWorkflow, createStep } from "@mastra/core/workflows"; +import { z } from "zod"; +import { readdir, readFile } from "fs/promises"; +import { join } from "path"; +import { exec } from "child_process"; +import { promisify } from "util"; +import { randomUUID } from "crypto"; +import { getRedisClient } from "../../redis/client"; + +const execAsync = promisify(exec); +const MAX_ITERATIONS = 5; + +const LOG_PREFIX = "[WORKFLOW]"; +function log(msg: string) { + const ts = new Date().toISOString(); + console.log(`${LOG_PREFIX} [${ts}] ${msg}`); +} + +async function discoverSourceFiles(repoPath: string): Promise { + const entries = await readdir(repoPath, { recursive: true }) as string[]; + return entries + .filter(f => + (f.endsWith(".ts") || f.endsWith(".js")) && + !f.includes("node_modules") && + !f.includes("__tests__") && + !f.includes(".d.ts") && + !f.includes("seeds/") && + !f.includes("migrations/") && + !f.includes("public/") + ) + .slice(0, 5); +} + +async function discoverTestFiles(repoPath: string): Promise { + const dirs = ["src/__tests__", "tests/integration", "tests/e2e"]; + const testFiles: string[] = []; + for (const dir of dirs) { + try { + const entries = await readdir(join(repoPath, dir), { recursive: true }) as string[]; + for (const entry of entries) { + if (entry.endsWith(".test.ts") || entry.endsWith(".test.js")) { + testFiles.push(`${dir}/${entry}`); + } + } + } catch { /* skip */ } + } + return testFiles; +} + +async function collectGeneratedFiles(repoPath: string): Promise<{ path: string; content: string }[]> { + const files: { path: string; content: string }[] = []; + const dirs = ["src/__tests__", "tests/integration", "tests/e2e"]; + for (const dir of dirs) { + try { + const entries = await readdir(join(repoPath, dir), { recursive: true }) as string[]; + for (const entry of entries) { + if (entry.endsWith(".test.ts") || entry.endsWith(".test.js")) { + const fullPath = join(repoPath, dir, entry); + const content = await readFile(fullPath, "utf-8"); + files.push({ path: `${dir}/${entry}`, content }); + } + } + } catch { /* skip */ } + } + return files; +} + +const FAILURE_PATTERN = /✗\s+(.+)\n[\s\S]*?Error:\s+(.+)/g; +function parseFailures(output: string) { + const failures: { testName: string; error: string }[] = []; + let match; + while ((match = FAILURE_PATTERN.exec(output)) !== null) { + failures.push({ testName: match[1].trim(), error: match[2].trim() }); + } + return failures; +} + +async function runTestFile(testFile: string, repoPath: string, iteration: number) { + const redis = getRedisClient(); + try { + const { stdout, stderr } = await execAsync( + `npx vitest run ${testFile} --reporter=verbose`, + { cwd: repoPath } + ); + const output = stdout + stderr; + const passed = !output.includes("FAIL") && !output.includes("failed"); + const failures = parseFailures(output); + const id = randomUUID(); + await redis.set(`test_results:${id}`, JSON.stringify({ + id, testId: testFile, filePath: testFile, passed, output, failures, + runAt: new Date().toISOString(), iteration, + })); + return { passed, output, failures }; + } catch (err: any) { + const output = (err.stdout || "") + (err.stderr || ""); + const failures = parseFailures(output); + const id = randomUUID(); + await redis.set(`test_results:${id}`, JSON.stringify({ + id, testId: testFile, filePath: testFile, passed: false, output, failures, + runAt: new Date().toISOString(), iteration, + })); + return { passed: false, output, failures }; + } +} + +// ── Step 1: Discover source files ───────────────────────────── +const discoverFilesStep = createStep({ + id: "discoverFiles", + description: "Scans the repository for source files that need tests", + inputSchema: z.object({ + repoPath: z.string(), + files: z.array(z.string()).optional(), + }), + outputSchema: z.object({ + sourceFiles: z.array(z.string()), + }), + execute: async ({ inputData }) => { + log("Discovering source files..."); + const sourceFiles = inputData.files ?? await discoverSourceFiles(inputData.repoPath); + log(`Found ${sourceFiles.length} source files`); + return { sourceFiles }; + }, +}); + +// ── Step 2: Research + generate + execute tests ────────────── +const researchAndGenerateStep = createStep({ + id: "researchAndGenerate", + description: "For each source file, calls the researchTestAgent to research, generate tests, run them, and store results in Redis", + inputSchema: z.object({ + sourceFiles: z.array(z.string()), + }), + outputSchema: z.object({ + testFiles: z.array(z.string()), + }), + execute: async ({ inputData, mastra, getInitData }) => { + const initData = getInitData() as { repoPath: string }; + const agent = mastra?.getAgent("researchTestAgent"); + if (!agent) throw new Error("researchTestAgent not found"); + + log(`Generating and running tests for ${inputData.sourceFiles.length} files...`); + + for (const file of inputData.sourceFiles) { + const testType = (file.includes("routes") || file.includes("api") || file.includes("service") || + file.includes("controller") || file.includes("middleware") || file.includes("handler")) + ? "INTEGRATION" + : (file.includes("app") || file.includes("server") || file.includes("index") || file.includes("auth")) + ? "E2E" + : "UNIT"; + + const ext = file.endsWith(".ts") ? ".ts" : ".js"; + const baseName = file.replace(/^src\//, "").replace(/\.(ts|js)$/, ""); + const testDir = testType === "UNIT" ? "src/__tests__" + : testType === "INTEGRATION" ? "tests/integration" + : "tests/e2e"; + + log(`Processing: ${file} → ${testType} tests`); + + const prompt = ` + Do the following steps in order for file "${file}": + + ## RESEARCH + 1. Call fetch-analysis with filePath="${file}" to get stored analysis context (if any). + 2. Call read-file with path="${file}" to read the source code. + + ## GENERATE ${testType} TESTS + 3. Write comprehensive vitest ${testType.toLowerCase()} tests covering: + - Happy path (normal expected behavior) + - Edge cases (empty input, nulls, boundary values) + - Error cases (exceptions, invalid input) + 4. Call write-file with: + - path="${testDir}/${baseName}.test${ext}" + - content = the full test file content + 5. Call store-tests with: + - filePath="${file}" + - testFilePath="${testDir}/${baseName}.test${ext}" + - testCode = the full test file content + + ## EXECUTE + 6. Call run-tests with testFilePath="${testDir}/${baseName}.test${ext}" + 7. Call store-results with: + - testId = "${file}" + - filePath = "${testDir}/${baseName}.test${ext}" + - passed = true or false (from run-tests) + - output = the full run-tests output + - failures = the array of failures from run-tests + - iteration = 1 + + Do all 7 steps now. + `; + await agent.generate(prompt); + log(` Completed: ${file}`); + } + + const testFiles = await discoverTestFiles(initData.repoPath); + log(`Generated ${testFiles.length} test files total`); + return { testFiles }; + }, +}); + +// ── Step 3: Fix + retest loop body ──────────────────────────── +const checkAndFixStep = createStep({ + id: "checkAndFix", + description: "Checks Redis for test results, calls editor agent to fix failures, then retests", + inputSchema: z.object({ + testFiles: z.array(z.string()), + }), + outputSchema: z.object({ + allPassed: z.boolean(), + iteration: z.number(), + }), + execute: async ({ inputData, mastra, getInitData, state, setState }) => { + const initData = getInitData() as { repoPath: string }; + const redis = getRedisClient(); + const currentIter = (state as any)?.iteration ?? 1; + + log(`Checking results for iteration ${currentIter}...`); + + const keys = await redis.keys("test_results:*"); + const iterationResults: { passed: boolean }[] = []; + for (const key of keys) { + const raw = await redis.get(key); + if (raw) { + const entry = JSON.parse(raw); + if (entry.iteration === currentIter) { + iterationResults.push({ passed: entry.passed }); + } + } + } + + const anyResults = iterationResults.length > 0; + const allPassed = anyResults && iterationResults.every(r => r.passed); + log(`Iteration ${currentIter}: ${iterationResults.length} test runs, allPassed=${allPassed}`); + + if (allPassed || currentIter >= MAX_ITERATIONS) { + await setState({ ...(state ?? {}), iteration: currentIter, allPassed }); + return { allPassed, iteration: currentIter }; + } + + // Fix failures via editor agent + log(`Editor agent fixing failures (iteration ${currentIter})...`); + const editor = mastra?.getAgent("editorAgent"); + if (editor) { + await editor.generate(` + Do the following steps in order: + 1. Call fetch-results with iteration=${currentIter} to get failing tests. + 2. For each failing test, call read-file on the source file being tested. + 3. For each failing test, fix the source file and call write-file to save it with: + - patchDescription = a short description of what you fixed + - iteration = ${currentIter} + Do all steps now. + `); + } + log("Editor fixes applied"); + + // Retest all test files + const nextIter = currentIter + 1; + log(`Re-running ${inputData.testFiles.length} test files (iteration ${nextIter})...`); + for (const testFile of inputData.testFiles) { + const result = await runTestFile(testFile, initData.repoPath, nextIter); + log(` ${testFile}: ${result.passed ? "PASSED" : "FAILED"}`); + } + + await setState({ ...(state ?? {}), iteration: nextIter }); + return { allPassed: false, iteration: nextIter }; + }, +}); + +// ── Step 4: Create GitHub PR ────────────────────────────────── +const createPRStep = createStep({ + id: "createPR", + description: "Creates a GitHub pull request with the generated tests and fixes", + inputSchema: z.object({ + allPassed: z.boolean(), + iteration: z.number(), + }), + outputSchema: z.object({ + prUrl: z.string().nullable(), + }), + execute: async ({ getInitData, state }) => { + const initData = getInitData() as { + repoPath: string; + githubToken?: string; + githubRepo?: string; + githubRef?: string; + githubSha?: string; + }; + const st = state as any; + const finalIteration = st.iteration ?? 0; + const allPassed = st.allPassed ?? false; + const status = allPassed ? "passed" : (finalIteration >= MAX_ITERATIONS ? "max_iterations" : "failed"); + log(`Final status: ${status} after ${finalIteration} iterations`); + + const { githubToken, githubRepo, githubRef, githubSha, repoPath } = initData; + if (!githubToken || !githubRepo) { + log("GITHUB_TOKEN or GITHUB_REPOSITORY not set — skipping PR"); + return { prUrl: null }; + } + + const [owner, repo] = githubRepo.split("/"); + const baseBranch = (githubRef ?? "").replace("refs/heads/", "").replace("refs/pull/", "").replace("/merge", ""); + if (!baseBranch) { + log("Could not determine base branch — skipping PR"); + return { prUrl: null }; + } + + const prBranch = `lemon/test-fix-${Date.now()}`; + + const files = await collectGeneratedFiles(repoPath); + if (files.length === 0) { + log("No generated test files found — skipping PR"); + return { prUrl: null }; + } + + try { + const branchRes = await fetch( + `https://api.github.com/repos/${owner}/${repo}/branches/${baseBranch}`, + { headers: { Authorization: `Bearer ${githubToken}`, Accept: "application/vnd.github+json" } } + ); + if (!branchRes.ok) throw new Error("Failed to get base branch"); + const branchData: any = await branchRes.json(); + const baseSha = branchData.commit.sha; + + const createRefRes = await fetch( + `https://api.github.com/repos/${owner}/${repo}/git/refs`, + { + method: "POST", + headers: { + Authorization: `Bearer ${githubToken}`, + Accept: "application/vnd.github+json", + "Content-Type": "application/json", + }, + body: JSON.stringify({ ref: `refs/heads/${prBranch}`, sha: baseSha }), + } + ); + if (!createRefRes.ok) { + const err = await createRefRes.text(); + throw new Error(`Failed to create branch: ${err}`); + } + log(`Branch created: ${prBranch}`); + + for (const file of files) { + const shaRes = await fetch( + `https://api.github.com/repos/${owner}/${repo}/contents/${file.path}?ref=${prBranch}`, + { headers: { Authorization: `Bearer ${githubToken}`, Accept: "application/vnd.github+json" } } + ); + const existingSha = shaRes.ok ? (await shaRes.json() as any).sha : undefined; + + const uploadRes = await fetch( + `https://api.github.com/repos/${owner}/${repo}/contents/${file.path}`, + { + method: "PUT", + headers: { + Authorization: `Bearer ${githubToken}`, + Accept: "application/vnd.github+json", + "Content-Type": "application/json", + }, + body: JSON.stringify({ + message: `🍋 lemon: generated ${file.path}`, + content: Buffer.from(file.content).toString("base64"), + branch: prBranch, + sha: existingSha ?? undefined, + }), + } + ); + if (!uploadRes.ok) { + const err = await uploadRes.text(); + log(`Failed to upload ${file.path}: ${err}`); + } else { + log(` Uploaded: ${file.path}`); + } + } + + const prBody = `## 🍋 lemon — AI Test Report + +**Branch:** ${baseBranch} +**Commit:** ${(githubSha ?? "").slice(0, 7) || "unknown"} + +### Test Results +| Status | Iterations | +|---|---| +| ${status} | ${finalIteration} | + +### Changed files +${files.map(f => `- \`${f.path}\``).join("\n")} +`; + + const prRes = await fetch(`https://api.github.com/repos/${owner}/${repo}/pulls`, { + method: "POST", + headers: { + Authorization: `Bearer ${githubToken}`, + Accept: "application/vnd.github+json", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }, + body: JSON.stringify({ + title: `🍋 lemon: auto-generated tests + fixes for ${baseBranch}`, + body: prBody, + head: prBranch, + base: baseBranch, + }), + }); + + if (!prRes.ok) { + const err = await prRes.text(); + log(`Failed to open PR: ${err}`); + return { prUrl: null }; + } + + const prData: any = await prRes.json(); + log(`PR created: ${prData.html_url}`); + return { prUrl: prData.html_url }; + } catch (err: any) { + log(`PR creation failed: ${err.message}`); + return { prUrl: null }; + } + }, +}); + +// ── Workflow composition ────────────────────────────────────── +export const testFixWorkflow = createWorkflow({ + id: "testFixWorkflow", + inputSchema: z.object({ + repoPath: z.string(), + githubToken: z.string().optional(), + githubRepo: z.string().optional(), + githubRef: z.string().optional(), + githubSha: z.string().optional(), + files: z.array(z.string()).optional(), + }), + outputSchema: z.object({ + prUrl: z.string().nullable(), + }), +}) + .then(discoverFilesStep) + .then(researchAndGenerateStep) + .dountil(checkAndFixStep, async ({ inputData }) => { + return inputData.allPassed === true || inputData.iteration >= MAX_ITERATIONS; + }) + .then(createPRStep) + .commit(); diff --git a/src/webhook-server.ts b/src/webhook-server.ts index b744627..c66004c 100644 --- a/src/webhook-server.ts +++ b/src/webhook-server.ts @@ -14,104 +14,9 @@ app.use(express.json()); const PORT = process.env.WEBHOOK_PORT ?? 3456; const WEBHOOK_SECRET = process.env.WEBHOOK_SECRET ?? ""; -const MAX_ITERATIONS = 5; const WORK_DIR = join(tmpdir(), "lemonx-workspaces"); const GITHUB_TOKEN = process.env.GITHUB_TOKEN ?? ""; -async function getFileSha(owner: string, repo: string, path: string, branch: string): Promise { - try { - const res = await fetch( - `https://api.github.com/repos/${owner}/${repo}/contents/${path}?ref=${branch}`, - { - headers: { - Authorization: `Bearer ${GITHUB_TOKEN}`, - Accept: "application/vnd.github+json", - }, - } - ); - if (!res.ok) return null; - const data = await res.json(); - return data.sha; - } catch { - return null; - } -} - -async function uploadFileToGitHub( - owner: string, - repo: string, - filePath: string, - content: string, - branch: string -): Promise { - const sha = await getFileSha(owner, repo, filePath, branch); - - const res = await fetch( - `https://api.github.com/repos/${owner}/${repo}/contents/${filePath}`, - { - method: "PUT", - headers: { - Authorization: `Bearer ${GITHUB_TOKEN}`, - Accept: "application/vnd.github+json", - "Content-Type": "application/json", - }, - body: JSON.stringify({ - message: `🍋 lemonx: generated ${filePath}`, - content: Buffer.from(content).toString("base64"), - branch, - sha: sha ?? undefined, - }), - } - ); - - if (!res.ok) { - const err = await res.text(); - throw new Error(`Failed to upload ${filePath}: ${err}`); - } -} - -async function getBaseBranchSha(owner: string, repo: string, branch: string): Promise { - try { - const res = await fetch( - `https://api.github.com/repos/${owner}/${repo}/branches/${branch}`, - { - headers: { - Authorization: `Bearer ${GITHUB_TOKEN}`, - Accept: "application/vnd.github+json", - }, - } - ); - if (!res.ok) return null; - const data = await res.json(); - return data.commit.sha; - } catch { - return null; - } -} - -async function createBranch(owner: string, repo: string, branch: string, baseSha: string): Promise { - const res = await fetch( - `https://api.github.com/repos/${owner}/${repo}/git/refs`, - { - method: "POST", - headers: { - Authorization: `Bearer ${GITHUB_TOKEN}`, - Accept: "application/vnd.github+json", - "Content-Type": "application/json", - }, - body: JSON.stringify({ - ref: `refs/heads/${branch}`, - sha: baseSha, - }), - } - ); - - if (!res.ok) { - const err = await res.text(); - throw new Error(`Failed to create branch ${branch}: ${err}`); - } -} - // ── Webhook signature verification ────────────────────────────── async function verifySignature(req: Request): Promise { if (!WEBHOOK_SECRET) return true; @@ -177,8 +82,6 @@ async function openPR(repoUrl: string, branch: string, prBranch: string, prTitle } const [, owner, repo] = match; - console.log(` 📝 Opening PR: ${prTitle}`); - const res = await fetch(`https://api.github.com/repos/${owner}/${repo}/pulls`, { method: "POST", headers: { @@ -201,29 +104,33 @@ async function openPR(repoUrl: string, branch: string, prBranch: string, prTitle return null; } - const data = await res.json(); + const data: any = await res.json(); console.log(` ✅ PR created: ${data.html_url}`); return data.html_url; } // ── Health check ──────────────────────────────────────────────── app.get("/health", (_req: Request, res: Response) => { - res.json({ status: "ok", agents: ["testGeneratorAgent", "executorAgent", "editorAgent", "integrationGeneratorAgent", "e2eGeneratorAgent"] }); + res.json({ + status: "ok", + agents: ["researchTestAgent", "editorAgent"], + workflows: ["testFixWorkflow"], + }); }); -// ── Trigger full test-fix loop (SYNCHRONOUS) ──────────────────── -app.post("/webhook/test-and-fix", async (req: Request, res: Response) => { +// ── Consolidated generate-and-test endpoint ──────────────────── +const generateAndTestHandler = async (req: Request, res: Response) => { if (!(await verifySignature(req))) { return res.status(401).json({ error: "Invalid signature" }); } - const { repoUrl, branch, commitSha } = req.body; + const { repoUrl, branch, commitSha, files, testType } = req.body; if (!repoUrl || !branch) { return res.status(400).json({ error: "repoUrl and branch are required" }); } - console.log(`\n🔔 Webhook received: ${repoUrl}/${branch} (${commitSha?.slice(0, 7) ?? "unknown"})`); + console.log(`\n🔔 Webhook received: generate-and-test for ${repoUrl}/${branch} (type: ${testType ?? "all"})`); let workspace: string | null = null; try { @@ -231,120 +138,37 @@ app.post("/webhook/test-and-fix", async (req: Request, res: Response) => { process.env.LEMON_WORKSPACE = workspace; console.log(` 📂 Working directory: ${workspace}`); - const unitResult = await runTestFixLoop(workspace, "unit"); - const integrationResult = await runTestFixLoop(workspace, "integration"); - const e2eResult = await runTestFixLoop(workspace, "e2e"); - console.log("\n✅ Test-fix loops completed:", { unit: unitResult, integration: integrationResult, e2e: e2eResult }); + const workflow = mastra.getWorkflow("testFixWorkflow"); + const result = await workflow.start({ + inputData: { + repoPath: workspace, + files: files ?? undefined, + githubToken: GITHUB_TOKEN, + githubRef: branch, + githubSha: commitSha, + }, + }); + + const output = result?.output as { prUrl?: string | null } | undefined; + console.log("\n✅ Test-fix workflow completed"); - const allPassed = unitResult.status === "passed" && integrationResult.status === "passed" && e2eResult.status === "passed"; + // Collect changed files for the summary + let changedFiles: string[] = []; + try { + const { stdout } = await execAsync("git diff --name-only HEAD", { cwd: workspace }); + changedFiles = stdout.trim().split("\n").filter(Boolean); + } catch { /* ignore */ } - // Build summary for CircleCI - const summary: { - status: string; - repo: string; - branch: string; - commit: string | undefined; - unit: { status: string; iterations: number; files: number }; - integration: { status: string; iterations: number; files: number }; - e2e: { status: string; iterations: number; files: number }; - changedFiles: string[]; - pr?: string; - } = { - status: allPassed ? "passed" : "failed", + res.json({ + status: "ok", repo: repoUrl, branch, commit: commitSha, - unit: { status: unitResult.status, iterations: unitResult.iterations, files: unitResult.files }, - integration: { status: integrationResult.status, iterations: integrationResult.iterations, files: integrationResult.files }, - e2e: { status: e2eResult.status, iterations: e2eResult.iterations, files: e2eResult.files }, - changedFiles: [ - ...(unitResult.changedFiles || []), - ...(integrationResult.changedFiles || []), - ...(e2eResult.changedFiles || []), - ], - }; - - // Open PR if all passed and there are changes - if (allPassed && summary.changedFiles.length > 0 && workspace) { - const prBranch = `lemonx/test-fix-${Date.now()}`; - const prTitle = `🍋 lemonx: auto-generated tests + fixes for ${branch}`; - const prBody = `## 🍋 lemonx — AI Test Report - -**Branch:** ${branch} -**Commit:** ${commitSha?.slice(0, 7) ?? "unknown"} - -### Test Results -| Test Type | Status | Iterations | -|---|---|---| -| Unit | ${unitResult.status} | ${unitResult.iterations} | -| Integration | ${integrationResult.status} | ${integrationResult.iterations} | -| E2E | ${e2eResult.status} | ${e2eResult.iterations} | - -### What changed -- Generated vitest unit, integration, and E2E tests for source files -- Ran tests and collected pass/fail results -- Applied code fixes to make tests pass -- All tests passing ✅ - -### Changed files -${summary.changedFiles.map((f: string) => `- \`${f}\``).join("\n")} -`; - - try { - const match = repoUrl.replace(/\.git$/, "").match(/github\.com[:/]([^/]+)\/([^/]+)/); - if (!match) { - console.log(" ⚠️ Could not parse repo URL for upload"); - } else { - const [, owner, repo] = match; - const testDirs = ["src/__tests__", "tests/integration", "tests/e2e"]; - const files: { path: string; content: string }[] = []; - - for (const dir of testDirs) { - try { - const entries = await readdir(join(workspace, dir), { recursive: true }) as string[]; - for (const entry of entries) { - if (entry.endsWith(".test.ts") || entry.endsWith(".test.js")) { - const fullPath = join(workspace, dir, entry); - const content = await readFile(fullPath, "utf-8"); - files.push({ path: `${dir}/${entry}`, content }); - } - } - } catch { - // Directory doesn't exist, skip - } - } - - if (files.length > 0) { - const baseSha = await getBaseBranchSha(owner, repo, branch); - if (!baseSha) { - console.log(" ❌ Could not get base branch SHA"); - } else { - await createBranch(owner, repo, prBranch, baseSha); - console.log(` 🌿 Created branch: ${prBranch}`); - - console.log(` 📤 Uploading ${files.length} files to GitHub...`); - for (const file of files) { - await uploadFileToGitHub(owner, repo, file.path, file.content, prBranch); - console.log(` ✓ ${file.path}`); - } - - const prUrl = await openPR(repoUrl, branch, prBranch, prTitle, prBody); - if (prUrl) { - summary.pr = prUrl; - console.log(`\n🎉 PR opened: ${prUrl}`); - } - } - } - } - } catch (err) { - console.log(` ⚠️ PR creation skipped: ${err}`); - } - } - - // Return results synchronously to CircleCI - res.json(summary); + prUrl: output?.prUrl ?? null, + changedFiles, + }); } catch (err: any) { - console.error("\n❌ Test-fix loop failed:", err); + console.error("\n❌ Workflow execution failed:", err); res.status(500).json({ status: "error", error: err.message, @@ -359,439 +183,15 @@ ${summary.changedFiles.map((f: string) => `- \`${f}\``).join("\n")} delete process.env.LEMON_WORKSPACE; } } -}); - -// ── Trigger unit test generation ──────────────────────────────── -app.post("/webhook/generate-tests", async (req: Request, res: Response) => { - if (!(await verifySignature(req))) { - return res.status(401).json({ error: "Invalid signature" }); - } - - const { repoUrl, branch, commitSha, files } = req.body; - - if (!repoUrl || !branch) { - return res.status(400).json({ error: "repoUrl and branch are required" }); - } - - console.log(`\n🔔 Webhook received: generate unit tests for ${repoUrl}/${branch}`); - - let workspace: string | null = null; - try { - workspace = await cloneRepo(repoUrl, branch, commitSha); - process.env.LEMON_WORKSPACE = workspace; - - const sourceFiles = files ?? await discoverFiles(workspace); - const generated = []; - - for (const file of sourceFiles) { - console.log(` Generating tests for: ${file}`); - const generator = mastra.getAgent("testGeneratorAgent"); - const testPath = file.replace(/^src\//, "").replace(/\.ts$/, ".test.ts"); - await generator.generate(` - Do the following steps in order: - 1. Call read-file with path="${file}" to read the source code. - 2. Write a comprehensive vitest unit test file for this source file. - 3. Call write-file with: - - path="src/__tests__/${testPath}" - - content = the full test file you wrote - 4. Call store-tests with: - - filePath="${file}" - - testFilePath="src/__tests__/${testPath}" - - testCode = the full test file content - Do all 4 steps now. - `); - generated.push({ file, testPath, success: true }); - console.log(` ✓ ${file} → src/__tests__/${testPath}`); - } - - res.json({ status: "done", generated }); - } catch (err: any) { - console.error(" ✗ Generation failed:", err); - res.status(500).json({ error: err.message }); - } finally { - if (workspace) { - await rm(workspace, { recursive: true, force: true }).catch(() => {}); - delete process.env.LEMON_WORKSPACE; - } - } -}); - -// ── Trigger integration test generation ───────────────────────── -app.post("/webhook/generate-integration-tests", async (req: Request, res: Response) => { - if (!(await verifySignature(req))) { - return res.status(401).json({ error: "Invalid signature" }); - } - - const { repoUrl, branch, commitSha, files } = req.body; - - if (!repoUrl || !branch) { - return res.status(400).json({ error: "repoUrl and branch are required" }); - } - - console.log(`\n🔔 Webhook received: generate integration tests for ${repoUrl}/${branch}`); - - let workspace: string | null = null; - try { - workspace = await cloneRepo(repoUrl, branch, commitSha); - process.env.LEMON_WORKSPACE = workspace; - - const sourceFiles = files ?? await discoverIntegrationFiles(workspace); - const generated = []; - - for (const file of sourceFiles) { - console.log(` Generating integration tests for: ${file}`); - const generator = mastra.getAgent("integrationGeneratorAgent"); - const testPath = file.replace(/^src\//, "").replace(/\.ts$/, ".test.ts"); - await generator.generate(` - Do the following steps in order: - 1. Call read-file with path="${file}" to read the source code. - 2. Write a comprehensive vitest integration test file for this source file. - 3. Call write-file with: - - path="tests/integration/${testPath}" - - content = the full test file you wrote - 4. Call store-tests with: - - filePath="${file}" - - testFilePath="tests/integration/${testPath}" - - testCode = the full test file content - Do all 4 steps now. - `); - generated.push({ file, testPath, success: true }); - console.log(` ✓ ${file} → tests/integration/${testPath}`); - } - - res.json({ status: "done", generated }); - } catch (err: any) { - console.error(" ✗ Generation failed:", err); - res.status(500).json({ error: err.message }); - } finally { - if (workspace) { - await rm(workspace, { recursive: true, force: true }).catch(() => {}); - delete process.env.LEMON_WORKSPACE; - } - } -}); - -// ── Trigger E2E test generation ───────────────────────────────── -app.post("/webhook/generate-e2e-tests", async (req: Request, res: Response) => { - if (!(await verifySignature(req))) { - return res.status(401).json({ error: "Invalid signature" }); - } - - const { repoUrl, branch, commitSha, files } = req.body; - - if (!repoUrl || !branch) { - return res.status(400).json({ error: "repoUrl and branch are required" }); - } - - console.log(`\n🔔 Webhook received: generate E2E tests for ${repoUrl}/${branch}`); - - let workspace: string | null = null; - try { - workspace = await cloneRepo(repoUrl, branch, commitSha); - process.env.LEMON_WORKSPACE = workspace; - - const sourceFiles = files ?? await discoverE2EFiles(workspace); - const generated = []; - - for (const file of sourceFiles) { - console.log(` Generating E2E tests for: ${file}`); - const generator = mastra.getAgent("e2eGeneratorAgent"); - const testPath = file.replace(/^src\//, "").replace(/\.ts$/, ".test.ts"); - await generator.generate(` - Do the following steps in order: - 1. Call read-file with path="${file}" to read the source code. - 2. Write a comprehensive vitest E2E test file for this source file. - 3. Call write-file with: - - path="tests/e2e/${testPath}" - - content = the full test file you wrote - 4. Call store-tests with: - - filePath="${file}" - - testFilePath="tests/e2e/${testPath}" - - testCode = the full test file content - Do all 4 steps now. - `); - generated.push({ file, testPath, success: true }); - console.log(` ✓ ${file} → tests/e2e/${testPath}`); - } - - res.json({ status: "done", generated }); - } catch (err: any) { - console.error(" ✗ Generation failed:", err); - res.status(500).json({ error: err.message }); - } finally { - if (workspace) { - await rm(workspace, { recursive: true, force: true }).catch(() => {}); - delete process.env.LEMON_WORKSPACE; - } - } -}); - -// ── Trigger test execution ────────────────────────────────────── -app.post("/webhook/run-tests", async (req: Request, res: Response) => { - if (!(await verifySignature(req))) { - return res.status(401).json({ error: "Invalid signature" }); - } - - const { repoUrl, branch, commitSha, testFile } = req.body; - - if (!repoUrl || !branch) { - return res.status(400).json({ error: "repoUrl and branch are required" }); - } - - let workspace: string | null = null; - try { - workspace = await cloneRepo(repoUrl, branch, commitSha); - process.env.LEMON_WORKSPACE = workspace; - - const testFiles = testFile - ? [testFile] - : await discoverTestFiles(workspace, "src/__tests__"); - - if (testFiles.length === 0) { - return res.json({ status: "done", results: [], message: "No test files found" }); - } +}; - const results = []; - for (const tf of testFiles) { - const executor = mastra.getAgent("executorAgent"); - const execRes = await executor.generate(` - Do the following steps in order: - 1. Call run-tests with testFilePath="${tf}" - 2. Call store-results with: - - testId = any unique string - - filePath = "${tf}" - - passed = true or false based on run-tests result - - output = the full output from run-tests - - failures = array of {testName, error} objects from the run-tests result - - iteration = 1 - Do both steps now. - `); - const passed = !execRes.text.toLowerCase().includes("fail") && !execRes.text.toLowerCase().includes("error"); - results.push({ file: tf, passed, summary: execRes.text.slice(0, 200) }); - } - - res.json({ status: "done", results }); - } catch (err: any) { - res.status(500).json({ error: err.message }); - } finally { - if (workspace) { - await rm(workspace, { recursive: true, force: true }).catch(() => {}); - delete process.env.LEMON_WORKSPACE; - } - } -}); - -// ── Helper: discover source files ─────────────────────────────── -async function discoverFiles(repoPath: string) { - const entries = await readdir(repoPath, { recursive: true }) as string[]; - return entries - .filter(f => - (f.endsWith(".ts") || f.endsWith(".js")) && - !f.includes("node_modules") && - !f.includes("__tests__") && - !f.includes(".d.ts") && - !f.includes("seeds/") && - !f.includes("migrations/") && - !f.includes("public/") - ) - .slice(0, 5); -} - -// ── Helper: discover test files ───────────────────────────────── -async function discoverTestFiles(repoPath: string, testDir: string) { - try { - const entries = await readdir(join(repoPath, testDir), { recursive: true }) as string[]; - return entries - .filter(f => f.endsWith(".test.ts") || f.endsWith(".test.js")) - .map(f => `${testDir}/${f}`); - } catch { - return []; - } -} - -// ── Full test-fix loop ────────────────────────────────────────── -async function runTestFixLoop(targetDir: string, testType: "unit" | "integration" | "e2e" = "unit") { - const executor = mastra.getAgent("executorAgent"); - const editor = mastra.getAgent("editorAgent"); - - const config = { - unit: { - generatorName: "testGeneratorAgent" as const, - testDir: "src/__tests__", - label: "unit", - discoverFn: () => discoverFiles(targetDir), - testPathFn: (file: string) => file.replace(/^src\//, "").replace(/\.ts$/, ".test.ts"), - promptFn: (file: string, testPath: string) => ` - Do the following steps in order: - 1. Call read-file with path="${file}" to read the source code. - 2. Write a comprehensive vitest unit test file for this source file. - 3. Call write-file with: - - path="src/__tests__/${testPath}" - - content = the full test file you wrote - 4. Call store-tests with: - - filePath="${file}" - - testFilePath="src/__tests__/${testPath}" - - testCode = the full test file content - Do all 4 steps now. - `, - }, - integration: { - generatorName: "integrationGeneratorAgent" as const, - testDir: "tests/integration", - label: "integration", - discoverFn: () => discoverIntegrationFiles(targetDir), - testPathFn: (file: string) => file.replace(/^src\//, "").replace(/\.ts$/, ".test.ts"), - promptFn: (file: string, testPath: string) => ` - Do the following steps in order: - 1. Call read-file with path="${file}" to read the source code. - 2. Write a comprehensive vitest integration test file for this source file. - 3. Call write-file with: - - path="tests/integration/${testPath}" - - content = the full test file you wrote - 4. Call store-tests with: - - filePath="${file}" - - testFilePath="tests/integration/${testPath}" - - testCode = the full test file content - Do all 4 steps now. - `, - }, - e2e: { - generatorName: "e2eGeneratorAgent" as const, - testDir: "tests/e2e", - label: "E2E", - discoverFn: () => discoverE2EFiles(targetDir), - testPathFn: (file: string) => file.replace(/^src\//, "").replace(/\.ts$/, ".test.ts"), - promptFn: (file: string, testPath: string) => ` - Do the following steps in order: - 1. Call read-file with path="${file}" to read the source code. - 2. Write a comprehensive vitest E2E test file for this source file. - 3. Call write-file with: - - path="tests/e2e/${testPath}" - - content = the full test file you wrote - 4. Call store-tests with: - - filePath="${file}" - - testFilePath="tests/e2e/${testPath}" - - testCode = the full test file content - Do all 4 steps now. - `, - }, - }; - -const cfg = config[testType]; - const generator = mastra.getAgent(cfg.generatorName); - - const files = await cfg.discoverFn(); - console.log(`🔍 Found ${files.length} source files for ${cfg.label} tests`); - - // Step 1: Generate tests - console.log(`📝 Generating ${cfg.label} tests...`); - for (const file of files) { - const testPath = cfg.testPathFn(file); - await generator.generate(cfg.promptFn(file, testPath)); - console.log(` ✓ ${file}`); - } - - // Step 2: Run + fix loop - for (let iteration = 1; iteration <= MAX_ITERATIONS; iteration++) { - console.log(`🧪 Iteration ${iteration}: Running ${cfg.label} tests...`); - - const testFiles = await discoverTestFiles(targetDir, cfg.testDir); - if (testFiles.length === 0) break; - - let allPassed = true; - for (const testFile of testFiles) { - const execRes = await executor.generate(` - Do the following steps in order: - 1. Call run-tests with testFilePath="${testFile}" - 2. Call store-results with: - - testId = any unique string - - filePath = "${testFile}" - - passed = true or false based on run-tests result - - output = the full output from run-tests - - failures = array of {testName, error} objects from the run-tests result - - iteration = ${iteration} - Do both steps now. - `); - console.log(` ${testFile}: ${execRes.text.slice(0, 80)}`); - - if (execRes.text.toLowerCase().includes("fail") || execRes.text.toLowerCase().includes("error")) { - allPassed = false; - } - } - - if (allPassed) { - console.log(`✅ All ${cfg.label} tests passed on iteration ${iteration}`); - const changedFiles = await getChangedFiles(targetDir); - return { status: "passed", iterations: iteration, files: testFiles.length, changedFiles, type: cfg.label }; - } - - if (iteration === MAX_ITERATIONS) { - console.log(`⚠️ Max iterations reached for ${cfg.label} tests`); - const changedFiles = await getChangedFiles(targetDir); - return { status: "max_iterations", iterations: iteration, files: testFiles.length, changedFiles, type: cfg.label }; - } - - console.log(`🔧 Iteration ${iteration}: Fixing ${cfg.label} failures...`); - const editRes = await editor.generate(` - Do the following steps in order: - 1. Call fetch-results with iteration=${iteration} to get failing tests. - 2. For each failing test, call read-file on the source file being tested. - 3. For each failing test, fix the source file and call write-file to save it with: - - patchDescription = a short description of what you fixed - - iteration = ${iteration} - Do all steps now. - `); - console.log(` Editor: ${editRes.text.slice(0, 150)}`); - } - - const changedFiles = await getChangedFiles(targetDir); - return { status: "completed", iterations: MAX_ITERATIONS, files: 0, changedFiles, type: cfg.label }; -} - -async function getChangedFiles(repoPath: string): Promise { - try { - const { stdout } = await execAsync("git diff --name-only HEAD", { cwd: repoPath }); - return stdout.trim().split("\n").filter(Boolean); - } catch { - return []; - } -} - -// ── Helper: discover integration test target files ────────────── -async function discoverIntegrationFiles(repoPath: string) { - const entries = await readdir(repoPath, { recursive: true }) as string[]; - return entries - .filter(f => - (f.endsWith(".ts") || f.endsWith(".js")) && - !f.includes("node_modules") && - !f.includes("__tests__") && - !f.includes(".d.ts") && - (f.includes("routes") || f.includes("api") || f.includes("service") || f.includes("controller") || f.includes("middleware") || f.includes("handler")) - ) - .slice(0, 5); -} - -// ── Helper: discover E2E test target files ────────────────────── -async function discoverE2EFiles(repoPath: string) { - const entries = await readdir(repoPath, { recursive: true }) as string[]; - return entries - .filter(f => - (f.endsWith(".ts") || f.endsWith(".js")) && - !f.includes("node_modules") && - !f.includes("__tests__") && - !f.includes(".d.ts") && - (f.includes("app") || f.includes("server") || f.includes("index") || f.includes("routes") || f.includes("auth")) - ) - .slice(0, 3); -} +app.post("/webhook/generate-and-test", generateAndTestHandler); +app.post("/webhook/test-and-fix", generateAndTestHandler); // ── Start server ──────────────────────────────────────────────── app.listen(PORT, () => { console.log(`🍋 lemon.test webhook server running on port ${PORT}`); - console.log(` POST /webhook/test-and-fix — full generate + run + fix loop (sync)`); - console.log(` POST /webhook/generate-tests — generate unit tests`); - console.log(` POST /webhook/generate-integration-tests — generate integration tests`); - console.log(` POST /webhook/generate-e2e-tests — generate E2E tests`); - console.log(` POST /webhook/run-tests — run tests`); - console.log(` GET /health — health check`); + console.log(` POST /webhook/generate-and-test — full generate + run + fix loop`); + console.log(` POST /webhook/test-and-fix — alias for /webhook/generate-and-test`); + console.log(` GET /health — health check`); }); diff --git a/tests/e2e/webhook-server.test.ts b/tests/e2e/webhook-server.test.ts index bcc06d1..097180a 100644 --- a/tests/e2e/webhook-server.test.ts +++ b/tests/e2e/webhook-server.test.ts @@ -15,10 +15,10 @@ describe("E2E: Webhook Server", () => { app.use(express.json()); app.get("/health", (_req, res) => { - res.json({ status: "ok", agents: ["testGeneratorAgent", "executorAgent", "editorAgent"] }); + res.json({ status: "ok", agents: ["researchTestAgent", "editorAgent"], workflows: ["testFixWorkflow"] }); }); - app.post("/webhook/test-and-fix", async (req, res) => { + app.post("/webhook/generate-and-test", async (req, res) => { const sig = req.headers["x-webhook-signature"] as string; if (!sig) { return res.status(401).json({ error: "Invalid signature" }); @@ -33,28 +33,35 @@ describe("E2E: Webhook Server", () => { return res.status(401).json({ error: "Invalid signature" }); } - const { targetDir } = req.body; - if (!targetDir) { - return res.status(400).json({ error: "targetDir is required" }); + const { repoUrl, branch } = req.body; + if (!repoUrl || !branch) { + return res.status(400).json({ error: "repoUrl and branch are required" }); } - res.json({ status: "accepted", message: "Test-fix loop started" }); + res.json({ status: "ok", prUrl: null, changedFiles: [] }); }); - app.post("/webhook/generate-tests", async (req, res) => { - const { targetDir } = req.body; - if (!targetDir) { - return res.status(400).json({ error: "targetDir is required" }); + app.post("/webhook/test-and-fix", async (req, res) => { + const sig = req.headers["x-webhook-signature"] as string; + if (!sig) { + return res.status(401).json({ error: "Invalid signature" }); + } + const crypto = await import("crypto"); + const expected = crypto + .createHmac("sha256", process.env.WEBHOOK_SECRET!) + .update(JSON.stringify(req.body)) + .digest("hex"); + + if (sig !== `sha256=${expected}`) { + return res.status(401).json({ error: "Invalid signature" }); } - res.json({ status: "done", generated: [] }); - }); - app.post("/webhook/run-tests", async (req, res) => { - const { targetDir } = req.body; - if (!targetDir) { - return res.status(400).json({ error: "targetDir is required" }); + const { repoUrl, branch } = req.body; + if (!repoUrl || !branch) { + return res.status(400).json({ error: "repoUrl and branch are required" }); } - res.json({ status: "done", results: [] }); + + res.json({ status: "ok", prUrl: null, changedFiles: [] }); }); server = app.listen(0); @@ -74,9 +81,9 @@ describe("E2E: Webhook Server", () => { expect(res.status).toBe(200); const body = await res.json(); expect(body.status).toBe("ok"); - expect(body.agents).toContain("testGeneratorAgent"); - expect(body.agents).toContain("executorAgent"); + expect(body.agents).toContain("researchTestAgent"); expect(body.agents).toContain("editorAgent"); + expect(body.workflows).toContain("testFixWorkflow"); }); }); @@ -120,12 +127,12 @@ describe("E2E: Webhook Server", () => { }); expect(res.status).toBe(200); const data = await res.json(); - expect(data.status).toBe("accepted"); + expect(data.status).toBe("ok"); }); }); describe("Webhook Endpoints", () => { - it("should reject test-and-fix without targetDir", async () => { + it("should reject test-and-fix without repoUrl", async () => { const crypto = await import("crypto"); const body = JSON.stringify({ repoUrl: "https://github.com/test/repo" }); const sig = crypto @@ -143,18 +150,18 @@ describe("E2E: Webhook Server", () => { }); expect(res.status).toBe(400); const data = await res.json(); - expect(data.error).toBe("targetDir is required"); + expect(data.error).toBe("repoUrl and branch are required"); }); - it("should reject generate-tests without targetDir", async () => { + it("should reject generate-and-test without repoUrl", async () => { const crypto = await import("crypto"); - const body = JSON.stringify({ files: ["src/index.ts"] }); + const body = JSON.stringify({ branch: "main" }); const sig = crypto .createHmac("sha256", "test-secret") .update(body) .digest("hex"); - const res = await fetch(`${baseUrl}/webhook/generate-tests`, { + const res = await fetch(`${baseUrl}/webhook/generate-and-test`, { method: "POST", headers: { "Content-Type": "application/json", @@ -163,17 +170,19 @@ describe("E2E: Webhook Server", () => { body, }); expect(res.status).toBe(400); + const data = await res.json(); + expect(data.error).toBe("repoUrl and branch are required"); }); - it("should reject run-tests without targetDir", async () => { + it("should accept valid generate-and-test request", async () => { const crypto = await import("crypto"); - const body = JSON.stringify({ testFile: "src/__tests__/index.test.ts" }); + const body = JSON.stringify({ repoUrl: "https://github.com/test/repo", branch: "main" }); const sig = crypto .createHmac("sha256", "test-secret") .update(body) .digest("hex"); - const res = await fetch(`${baseUrl}/webhook/run-tests`, { + const res = await fetch(`${baseUrl}/webhook/generate-and-test`, { method: "POST", headers: { "Content-Type": "application/json", @@ -181,7 +190,7 @@ describe("E2E: Webhook Server", () => { }, body, }); - expect(res.status).toBe(400); + expect(res.status).toBe(200); }); }); });