From 434a5cd813dc113544414103716600248096405f Mon Sep 17 00:00:00 2001 From: akashgit Date: Sat, 29 Aug 2026 14:51:18 -0400 Subject: [PATCH 1/9] =?UTF-8?q?feat:=20add=20design-v2=20workflow=20?= =?UTF-8?q?=E2=80=94=20inference-time=20scaling=20for=20design=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the design-v2 workflow as a contributed workflow with dynamic research/strategy/QA directors that scale agent count at inference time based on project complexity, replacing the static fork/join patterns. - Research Director: decides N research directions dynamically (3-7) - Strategy Director: spawns M strategy perspectives (2-5) - QA Director: creates K tailored adversarial test approaches (2-5) - User Intent Ledger: tracks idea + feedback throughout the session - Design Doc: rewrites strategy into human-readable design document - Synthesize QA: merges adversarial reports with confidence scoring 29 nodes, 32 edges. 63 tests covering graph structure, node properties, edge wiring, post checks, and removed-node assertions. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../workflow/contributed/design_v2/README.md | 38 + .../contributed/design_v2/__init__.py | 3 + .../contributed/design_v2/test_workflow.py | 29 + .../contributed/design_v2/workflow.py | 811 ++++++++++++++++++ factory/workflow/definitions.py | 3 + tests/test_workflow_design_v2.py | 274 ++++++ workflow-design-v2/SKILL.annotations.yaml | 663 ++++++++++++++ workflow-design-v2/SKILL.md | 731 ++++++++++++++++ 8 files changed, 2552 insertions(+) create mode 100644 factory/workflow/contributed/design_v2/README.md create mode 100644 factory/workflow/contributed/design_v2/__init__.py create mode 100644 factory/workflow/contributed/design_v2/test_workflow.py create mode 100644 factory/workflow/contributed/design_v2/workflow.py create mode 100644 tests/test_workflow_design_v2.py create mode 100644 workflow-design-v2/SKILL.annotations.yaml create mode 100644 workflow-design-v2/SKILL.md diff --git a/factory/workflow/contributed/design_v2/README.md b/factory/workflow/contributed/design_v2/README.md new file mode 100644 index 000000000..1bf2275f9 --- /dev/null +++ b/factory/workflow/contributed/design_v2/README.md @@ -0,0 +1,38 @@ +# design-v2 Workflow + +Design mode with inference-time scaling: dynamic research, multi-strategy, user intent tracking, and parallel adversarial QA. + +## Graph + +``` +init_user_intent (FnNode) + → gate_has_factory (GateNode) + → [PROCEED] graph_update → study → graph_explorer → concat_study + → [HALT] discover → gate_factory_md_exists → factory_init → graph_update + → concat_study → research_director (AgentNode/CEO) + → strategy_director (AgentNode/CEO) + → synthesize_strategy (AgentNode/STRATEGIST) + → design_doc (AgentNode/STRATEGIST) + → gate_strategy (GateNode/user) + → [PROCEED] begin → builder → fork_qa + → [RELOOP] strategy_director + → fork_qa → health_checker, code_reviewer, qa_director → join_qa + → synthesize_qa (FnNode) → gate_qa (GateNode/agent) + → [PROCEED] gate_doc_freshness → gate_precheck → archivist_build + → [RELOOP] builder +``` + +## Key differences from design (v1) + +- **Research Director** replaces static fork/join research — dynamically decides N research directions +- **Strategy Director** replaces single strategist — spawns M strategy perspectives +- **QA Director** replaces single adversarial tester — spawns K tailored test approaches +- **User Intent Ledger** tracks the original idea and all feedback through the session +- **Design Doc** rewrites strategy into a human-readable design document +- **Synthesize QA** merges all adversarial reports with confidence scoring + +## Usage + +```bash +factory ceo /path/to/project --mode design-v2 +``` diff --git a/factory/workflow/contributed/design_v2/__init__.py b/factory/workflow/contributed/design_v2/__init__.py new file mode 100644 index 000000000..8a7feb27f --- /dev/null +++ b/factory/workflow/contributed/design_v2/__init__.py @@ -0,0 +1,3 @@ +from .workflow import meta, workflow + +__all__ = ["meta", "workflow"] diff --git a/factory/workflow/contributed/design_v2/test_workflow.py b/factory/workflow/contributed/design_v2/test_workflow.py new file mode 100644 index 000000000..202e95b4b --- /dev/null +++ b/factory/workflow/contributed/design_v2/test_workflow.py @@ -0,0 +1,29 @@ +"""In-package smoke tests for the design-v2 contributed workflow.""" + +from __future__ import annotations + +from factory.workflow.contributed.design_v2 import meta, workflow +from factory.workflow.definitions import register_all + + +class TestDesignV2Smoke: + def test_meta_name(self) -> None: + assert meta["name"] == "design-v2" + + def test_meta_description(self) -> None: + assert len(meta["description"]) > 0 + + def test_graph_validates(self) -> None: + wf = workflow() + issues = wf.validate_graph() + assert issues == [], f"design-v2 has validation issues: {issues}" + + def test_registered_in_register_all(self) -> None: + workflows = register_all() + assert "design-v2" in workflows + + def test_registered_workflow_valid(self) -> None: + workflows = register_all() + wf = workflows["design-v2"] + issues = wf.validate_graph() + assert issues == [], f"Registered design-v2 has issues: {issues}" diff --git a/factory/workflow/contributed/design_v2/workflow.py b/factory/workflow/contributed/design_v2/workflow.py new file mode 100644 index 000000000..ad1e5a628 --- /dev/null +++ b/factory/workflow/contributed/design_v2/workflow.py @@ -0,0 +1,811 @@ +"""design-v2: Design mode with inference-time scaling. + +Dynamic research, multi-strategy, user intent tracking, +and QA Director with tailored adversarial testing. +""" + +from __future__ import annotations + +from factory.workflow.definitions import _study_subgraph, build_workflow +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + ArtifactCheck, + Edge, + FnNode, + ForkNode, + GateNode, + JoinNode, + VerdictType, + Workflow, +) + +meta = { + "name": "design-v2", + "description": ( + "Design mode with inference-time scaling: dynamic research, " + "multi-strategy, user intent tracking, parallel adversarial QA" + ), +} + +# ── Prompt templates ──────────────────────────────────────────── + +RESEARCH_DIRECTOR_PROMPT = """\ +You are the Research Director for this design session. + +Read: +- `.factory/strategy/study-combined.md` — project study findings (observations + graph analysis) +- `.factory/strategy/user-intent.md` — the user's original idea + +Your task has TWO phases: + +PHASE 1 — DESIGN RESEARCH DIRECTIONS +Analyze the design space and identify N orthogonal research dimensions. +Default dimensions (adapt or replace based on the specific project): + - Similar projects / prior art + - Technology stack options + - Common pitfalls and failure modes +You may add dimensions specific to the domain (e.g., security, UX patterns, +data model constraints, compliance requirements, API design, concurrency). + +N is NOT fixed — YOU decide based on domain complexity: + - Simple CLI or library: 3 directions + - Web app with auth, database, UI: 4-5 directions + - Complex system with integrations, security, compliance: 5-7 directions + +For each direction, design a TAILORED prompt — not a generic template. +Bad: "Research similar projects and prior art" +Good: "Find existing link-checking tools that handle Obsidian-style wikilinks + ([[note]]) and image embeds (![[image.png]]). Compare how they resolve + relative paths vs vault-root-relative paths." + +Write the research plan to `.factory/strategy/research-plan.json`: +```json +[ + {"focus": "...", "slug": "...", "prompt": "..."} +] +``` + +Constraints: +- Minimum 3 directions, maximum 7 +- Each slug must be unique and kebab-case +- Prompts must be specific to THIS project, not generic templates + +PHASE 2 — EXECUTE RESEARCH +For each direction in the plan, spawn a researcher agent: +``` +factory agent researcher --task "" --project {project_path} +``` + +Each researcher writes to `.factory/strategy/research-.md`. + +After ALL researchers complete, review quality: +- Each research file exists and has substantive content (>50 bytes) +- No two reports cover the same ground excessively +- Key risks and opportunities are covered + +If a researcher produced thin output, re-invoke it with a more specific prompt. + +Write a brief research summary to the end of research-plan.json noting +which directions completed and any quality issues.""" + +STRATEGY_DIRECTOR_PROMPT = """\ +You are the Strategy Director for this design session. + +Read: +- ALL research reports at `.factory/strategy/research-*.md` +- `.factory/strategy/research-plan.json` — which research directions were explored +- `.factory/strategy/user-intent.md` — the user's original idea +- `.factory/strategy/study-combined.md` — project context + +Your task has TWO phases: + +PHASE 1 — DESIGN STRATEGY PERSPECTIVES +Analyze the research findings and user intent to identify M strategy +perspectives this project needs. + +Default perspectives (adapt or replace based on the specific project): + - Architecture strategy — how to build it (components, phases, tech choices) + - Testing/verification strategy — how to verify it works (acceptance criteria, + test plan, edge cases) + - Risk/scope strategy — what to cut, what's hard, what breaks + +You may add perspectives specific to the domain: + - Security strategy (for auth-heavy projects) + - Data modeling strategy (for data-heavy projects) + - API design strategy (for API-first projects) + - Performance strategy (for latency-sensitive systems) + +M is NOT fixed — YOU decide based on project complexity: + - Simple project: 2-3 perspectives + - Medium project: 3-4 perspectives + - Complex project: 4-5 perspectives + +For each perspective, design a TAILORED prompt. +Bad: "Create an architecture strategy" +Good: "Design the architecture for a markdown link checker CLI. The core + challenge is resolving Obsidian wikilinks against a configurable vault + root while also supporting standard URLs with redirect following. + Research shows handles HTTP well but nothing handles + wikilinks — design that component from scratch." + +Write the strategy plan to `.factory/strategy/strategy-plan.json`: +```json +[ + {"perspective": "...", "slug": "...", "prompt": "..."} +] +``` + +Constraints: +- Minimum 2 perspectives, maximum 5 +- Each slug must be unique and kebab-case +- One perspective MUST cover testing/verification with explicit acceptance criteria +- Prompts must reference specific findings from the research reports + +PHASE 2 — EXECUTE STRATEGIES +For each perspective in the plan, spawn a strategist agent: +``` +factory agent strategist --task "" --project {project_path} +``` + +Each strategist writes to `.factory/strategy/strategy-.md`. + +After ALL strategists complete, review quality: +- Each strategy file exists and has substantive content (>100 bytes) +- The testing strategy has a `### Acceptance Criteria` section with checkboxes +- Architecture strategy cites research findings +- No critical perspective is missing + +If a strategist produced thin output, re-invoke it with a more specific prompt. + +Write a brief strategy summary to the end of strategy-plan.json noting +which perspectives completed and any quality issues.""" + +SYNTHESIZE_STRATEGY_PROMPT = """\ +You are the Strategy Synthesizer. Compile one final plan from all +strategy inputs. + +Read: +- ALL strategy files at `.factory/strategy/strategy-*.md` +- `.factory/strategy/strategy-plan.json` — which perspectives were explored +- `.factory/strategy/user-intent.md` — ground truth for user's ask + +Write the final plan to `.factory/strategy/current.md`. + +Required sections (in this order): +### Architecture + Merge from architecture strategy. Include component list and interfaces. +### Phased Plan + #### Phase 1: + - **What:** + - **Why:** + - **Acceptance criteria:** + - **Risks:** + #### Phase 2: + ... +### Acceptance Criteria + Full checklist from testing strategy. Each item must be: + - [ ] Specific enough for pass/fail verification + - Traceable to user intent (cite which part of user-intent.md) +### MVP Scope + From risk strategy. In vs deferred. +### Deferred Features + Items requiring human intervention or explicitly deferred. + +CRITICAL: The ### Acceptance Criteria section is the contract between +the builder and QA. It flows to adversarial testers who verify each +criterion independently. Make every item testable and unambiguous.""" + +DESIGN_DOC_PROMPT = """\ +You are a Technical Writer and Design Architect. Your job: take the structured +strategy at `.factory/strategy/current.md` and rewrite it as a proper +DESIGN DOCUMENT — a document a human reviewer can read end-to-end and +understand exactly what is being built, why, and how. + +Read: +- `.factory/strategy/current.md` — the structured strategy (your raw input) +- `.factory/strategy/user-intent.md` — the user's original idea and feedback + +Rewrite `.factory/strategy/current.md` IN PLACE. Replace the compressed +bullet points with a well-structured design document. + +Required sections (in this order): + +## What We're Building + Explain the project in 2-3 paragraphs of prose. What does it do? + Who is it for? What problem does it solve? Reference the user's + original words from user-intent.md. + +## Architecture + Describe the system architecture in full sentences. + Include a text-based architecture diagram using box-drawing characters: + ``` + ┌──────────┐ ┌──────────┐ ┌──────────┐ + │ Component│────▶│ Component│────▶│ Component│ + └──────────┘ └──────────┘ └──────────┘ + ``` + Explain each component — what it does, why it exists, how it connects. + +## How It Works + Walk through the user flow step by step. For a CLI, show example + invocations and expected output. For a web app, describe the user + journey screen by screen. For a library, show usage examples. + + This should read like a tutorial — someone unfamiliar with the project + should be able to follow along. + +## Phased Plan + For each phase, explain: + - What gets built in this phase and why this ordering + - What the user can do after this phase completes + - How to verify the phase worked (concrete test commands or checks) + + Use prose paragraphs, not just bullet points. Each phase should + read as a self-contained "chapter." + +## Acceptance Criteria + Present the full checklist, but group criteria by category and add + context for each. Explain WHY each criterion matters, not just what it is. + + Format: category heading, then checkbox items with brief explanation. + +## MVP Scope + What's in, what's deferred, and why. Explain the tradeoffs. + +## Deferred Features + Items deferred to future phases, with brief rationale. + +CRITICAL RULES: +- Write in full sentences and paragraphs, NOT bullet points +- The reader should understand the design WITHOUT reading any other file +- Include concrete examples (CLI invocations, API calls, code snippets) +- Architecture diagrams must use text/box-drawing characters +- Every technical choice must be explained — no unexplained jargon +- The document must be self-contained: a human reviewer reads ONLY this + file and decides whether to approve the design""" + +QA_DIRECTOR_PROMPT = """\ +You are the QA Director for this design session. + +Read: +- `.factory/strategy/current.md` — the design document with acceptance criteria +- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for +- `.factory/reviews/builder-latest.md` — what the builder implemented + +Your task has TWO phases: + +PHASE 1 — DESIGN QA APPROACHES +Analyze the acceptance criteria, the user's intent, and the builder's +implementation to identify K orthogonal testing approaches. + +Default approaches (adapt or replace based on the specific project): + - Happy path verification — does each acceptance criterion pass with + normal, expected inputs? + - Edge case & boundary testing — empty inputs, large inputs, special + characters, off-by-one, boundary conditions + - User intent verification — does the output match what the user + ACTUALLY asked for (from user-intent.md), not just the plan? + +You may add approaches specific to the implementation: + - Security testing (for auth, file I/O, user-facing APIs) + - Integration testing (for multi-component systems) + - Performance testing (for latency-sensitive features) + - Error handling testing (for systems with many failure modes) + - Concurrency testing (for parallel/async systems) + +K is NOT fixed — YOU decide based on the complexity of the acceptance +criteria and the nature of the implementation: + - Simple implementation (3-5 criteria): 2 testers + - Medium implementation (5-10 criteria): 3 testers + - Complex implementation (10+ criteria, security, integrations): 4-5 testers + +For each approach, design a TAILORED prompt — not a generic template. +Bad: "Test the implementation for edge cases" +Good: "Test the Obsidian wikilink resolver with these edge cases: + nested vault folders (vault/sub/note.md linking to ../other.md), + wikilinks with display text ([[note|display]]), + wikilinks to non-existent files, wikilinks with anchor + fragments ([[note#heading]]), and case-sensitivity mismatches." + +Write the QA plan to `.factory/reviews/qa-plan.json`: +```json +[ + {"approach": "...", "slug": "...", "prompt": "..."} +] +``` + +Constraints: +- Minimum 2 approaches, maximum 5 +- Each slug must be unique and kebab-case +- ALL acceptance criteria from current.md must be covered by at least + one tester's prompt +- One approach MUST verify user intent against user-intent.md +- Prompts must reference specific acceptance criteria and implementation details + +PHASE 2 — EXECUTE QA +For each approach in the plan, spawn an adversarial tester agent: +``` +factory agent adversarial_tester --task "" --project {project_path} +``` + +Each tester writes to `.factory/reviews/adversarial--latest.md`. + +After ALL testers complete, review quality: +- Each adversarial report exists and has substantive findings +- All acceptance criteria are covered by at least one tester +- No tester missed its assigned focus area +- Critical findings are actually reproducible (spot-check) + +If a tester produced thin output, re-invoke it with a more specific prompt. + +Write a brief QA summary to the end of qa-plan.json noting which +approaches completed and any quality issues.""" + +ADVERSARIAL_PROMPT = """\ +You are an adversarial tester. Your job: break the implementation. + +Read: +- `.factory/strategy/current.md` — the design document and acceptance criteria +- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for +- `.factory/reviews/builder-latest.md` — builder's work summary +- Source code changes (use git diff) + +Procedure: +1. Read the acceptance criteria from current.md +2. For each criterion, attempt to verify it by running the code +3. Try edge cases, invalid inputs, boundary conditions +4. Check that the implementation matches user intent, not just the plan +5. Look for security issues, error handling gaps, missing validations + +Output format: +# Adversarial Test Report + +## Acceptance Criteria Verification +- [ ] Criterion 1: PASS/FAIL — evidence +- [ ] Criterion 2: PASS/FAIL — evidence + +## Edge Case Findings +- Finding: + - Steps to reproduce: + - Expected: + - Actual: + +## User Intent Verification +- Does the output match what the user asked for? Evidence: <...>""" + +GATE_QA_PROMPT = """\ +You are the CEO reviewing QA results. This is the final gate before merge. + +Read: +- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for +- `.factory/reviews/qa-synthesized.md` — merged QA report (health check, + code review, and synthesized adversarial findings) + +Decision framework: + +PROCEED if ALL of these hold: + 1. All acceptance criteria from current.md are verified PASS + 2. Health check passes (tests green, eval not regressed) + 3. No blocking code review issues + 4. No HIGH-confidence adversarial findings that violate user intent + +RELOOP to builder (max 3 iterations) if ANY of these hold: + 1. An acceptance criterion failed — cite which one + 2. Health check failed — cite which check + 3. Blocking review or HIGH adversarial findings + + When relooping, provide feedback mapped to SPECIFIC user requirements: + - "User asked for X (user-intent.md), but " + - "Acceptance criterion '' FAILED: " + + IMPORTANT: Append your reloop feedback to .factory/strategy/user-intent.md + under a new '## [timestamp] Reloop Feedback (Iteration N)' heading. + +HALT if: + - 3 reloops exhausted without resolution + - Fundamental design flaw that builder iterations cannot fix""" + + +def workflow() -> Workflow: + """Build the design-v2 workflow graph.""" + wf = build_workflow() + + # ── Bootstrap: gate_has_factory + discover + factory.md creation ── + + wf.nodes["gate_has_factory"] = GateNode( + id="gate_has_factory", + evaluator_type="fn", + evaluator_command=( + 'python3 -c "' + "from pathlib import Path; " + 'exists = Path("{project_path}/.factory/config.json").exists(); ' + 'print("PROCEED" if exists else "HALT")' + '"' + ), + ) + + wf.nodes["discover"] = FnNode( + id="discover", + command="factory discover {project_path}", + writes={".factory/eval_profile.json"}, + ) + + wf.nodes["gate_factory_md_exists"] = GateNode( + id="gate_factory_md_exists", + evaluator_type="fn", + evaluator_command=( + 'python3 -c "' + "from pathlib import Path; " + 'exists = Path("{project_path}/factory.md").exists(); ' + 'print("PROCEED" if exists else "HALT")' + '"' + ), + ) + + wf.nodes["create_factory_md"] = AgentNode( + id="create_factory_md", + role=AgentRole.CEO, + prompt_template=( + "Create factory.md from template. " + "Copy the factory config template to the project root. " + "Fill in: Goal, Scope, Guards, Eval command, Threshold, and Smoke Test. " + "If .factory/eval_spec.json exists, populate the Eval Spec section. " + "If .factory/strategy/current.md has a Research Configuration section, " + "populate research sections (Research Target, Mutable/Fixed Surfaces, etc.)." + ), + reads={".factory/eval_profile.json"}, + writes={"factory.md"}, + ) + + wf.nodes["factory_init"] = FnNode( + id="factory_init", + command="factory init {project_path}", + notes=( + "Parse factory.md and generate .factory/config.json. " + "Must run after factory.md is created." + ), + reads={"factory.md"}, + writes={".factory/config.json"}, + ) + + # ── Study subgraph ── + + s_nodes, s_edges = _study_subgraph() + wf.nodes.update(s_nodes) + + # ── User intent init (NEW) ── + + wf.nodes["init_user_intent"] = FnNode( + id="init_user_intent", + command=( + 'python3 -c "' + "import datetime, os; " + "project = '{project_path}'; " + "ts = datetime.datetime.now().isoformat(timespec='seconds'); " + "idea = os.environ.get('FOCUS', os.environ.get('FACTORY_IDEA', " + "'No idea provided')); " + "content = f'# User Intent Ledger\\n\\n## [{ts}] Initial Idea\\n" + "{idea}\\n'; " + "open(f'{project}/.factory/strategy/user-intent.md', 'w').write(" + "content); " + "print(f'User intent ledger initialized at {ts}')" + '"' + ), + writes={".factory/strategy/user-intent.md"}, + notes="Creates the user intent ledger with the initial idea.", + ) + + # ── Research Director (NEW — replaces fork/join research) ── + + wf.nodes["research_director"] = AgentNode( + id="research_director", + role=AgentRole.CEO, + timeout=3600, + prompt_template=RESEARCH_DIRECTOR_PROMPT, + reads={ + ".factory/strategy/study-combined.md", + ".factory/strategy/user-intent.md", + }, + writes={".factory/strategy/research-plan.json"}, + post_checks=[ + ArtifactCheck( + path=".factory/strategy/research-plan.json", + must_exist=True, + min_size=20, + ), + ], + ) + + # ── Strategy Director (NEW — replaces single strategist) ── + + wf.nodes["strategy_director"] = AgentNode( + id="strategy_director", + role=AgentRole.CEO, + timeout=3600, + prompt_template=STRATEGY_DIRECTOR_PROMPT, + reads={ + ".factory/strategy/research-plan.json", + ".factory/strategy/user-intent.md", + ".factory/strategy/study-combined.md", + }, + writes={".factory/strategy/strategy-plan.json"}, + post_checks=[ + ArtifactCheck( + path=".factory/strategy/strategy-plan.json", + must_exist=True, + min_size=20, + ), + ], + ) + + # ── Synthesize Strategy (NEW) ── + + wf.nodes["synthesize_strategy"] = AgentNode( + id="synthesize_strategy", + role=AgentRole.STRATEGIST, + prompt_template=SYNTHESIZE_STRATEGY_PROMPT, + reads={".factory/strategy/user-intent.md"}, + writes={".factory/strategy/current.md"}, + post_checks=[ + ArtifactCheck( + path=".factory/strategy/current.md", + must_exist=True, + min_size=200, + must_contain=[ + "### Phased Plan", + "### Acceptance Criteria", + ], + ), + ], + ) + + # ── Design Doc (NEW) ── + + wf.nodes["design_doc"] = AgentNode( + id="design_doc", + role=AgentRole.STRATEGIST, + prompt_template=DESIGN_DOC_PROMPT, + reads={ + ".factory/strategy/current.md", + ".factory/strategy/user-intent.md", + }, + writes={".factory/strategy/current.md"}, + post_checks=[ + ArtifactCheck( + path=".factory/strategy/current.md", + must_exist=True, + min_size=500, + must_contain=[ + "## What We're Building", + "## Architecture", + "## How It Works", + "## Acceptance Criteria", + ], + ), + ], + ) + + # ── QA Director (NEW — replaces static adversarial tester) ── + + wf.nodes["qa_director"] = AgentNode( + id="qa_director", + role=AgentRole.CEO, + timeout=3600, + prompt_template=QA_DIRECTOR_PROMPT, + reads={ + ".factory/strategy/current.md", + ".factory/strategy/user-intent.md", + ".factory/reviews/builder-latest.md", + }, + writes={".factory/reviews/qa-plan.json"}, + post_checks=[ + ArtifactCheck( + path=".factory/reviews/qa-plan.json", + must_exist=True, + min_size=20, + ), + ], + ) + + # ── Synthesize QA (NEW — glob-based merge of adversarial reports) ── + + wf.nodes["synthesize_qa"] = FnNode( + id="synthesize_qa", + command=( + 'python3 -c "' + "from pathlib import Path; " + "import re, glob; " + "project = '{project_path}'; " + "reports = []; " + "for p in sorted(Path(f'{project}/.factory/reviews').glob(" + "'adversarial-*-latest.md')): " + " slug = p.name.replace('-latest.md', '').replace(" + "'adversarial-', ''); " + " reports.append((slug, p.read_text())); " + "findings = {}; " + "for tester_slug, text in reports: " + " for line in text.splitlines(): " + " stripped = line.strip(); " + " if stripped.startswith('- ') or stripped.startswith('* '): " + " key = re.sub(r'\\s+', ' ', " + "stripped[2:].strip().lower()[:80]); " + " findings.setdefault(key, []).append(tester_slug); " + "high = [(k, v) for k, v in findings.items() if len(v) >= 2]; " + "medium = [(k, v) for k, v in findings.items() if len(v) == 1]; " + "out = ['# Synthesized QA Report\\n']; " + "hc = Path(f'{project}/.factory/reviews/health-check.md'); " + "cr = Path(f'{project}/.factory/reviews/code-review.md'); " + "out.append('## Health Check\\n'); " + "out.append(hc.read_text() if hc.exists() else '(not available)'); " + "out.append('\\n## Code Review\\n'); " + "out.append(cr.read_text() if cr.exists() else '(not available)'); " + "out.append('\\n## High-Confidence Adversarial Findings " + "(caught by 2+ testers)\\n'); " + "[out.append(f'- {k} (testers: {v})') for k, v in high]; " + "if not high: out.append('- (none)'); " + "out.append('\\n## Medium-Confidence Adversarial Findings " + "(single tester)\\n'); " + "[out.append(f'- {k} (tester: {v[0]})') for k, v in medium]; " + "if not medium: out.append('- (none)'); " + "out.append('\\n## Raw Adversarial Reports\\n'); " + "[out.append(f'### Tester: {slug}\\n{text}\\n') " + "for slug, text in reports]; " + "Path(f'{project}/.factory/reviews/qa-synthesized.md').write_text(" + "'\\n'.join(out)); " + "print(f'Synthesized {len(high)} high + {len(medium)} medium " + "findings from {len(reports)} adversarial reports')" + '"' + ), + reads={ + ".factory/reviews/health-check.md", + ".factory/reviews/code-review.md", + }, + writes={".factory/reviews/qa-synthesized.md"}, + notes=( + "Merges ALL adversarial-*-latest.md reports into one synthesized " + "QA report. Uses glob pattern — works for any K testers. " + "Findings caught by 2+ testers = HIGH confidence. " + "Single-source findings surfaced but marked. " + "Health checker and code reviewer reports included as pass-through." + ), + ) + + # ── Modify existing nodes ── + + wf.nodes["fork_qa"] = ForkNode( + id="fork_qa", + targets=["health_checker", "code_reviewer", "qa_director"], + ) + + wf.nodes["join_qa"] = JoinNode( + id="join_qa", + sources=["health_checker", "code_reviewer", "qa_director"], + reads={ + ".factory/reviews/health-check.md", + ".factory/reviews/code-review.md", + ".factory/reviews/qa-plan.json", + }, + ) + + wf.nodes["gate_strategy"] = GateNode( + id="gate_strategy", + evaluator_type="user", + reads={ + ".factory/strategy/current.md", + ".factory/strategy/user-intent.md", + }, + gate_prompt=( + "Review the design document at .factory/strategy/current.md. " + "This is a human-readable design document explaining what will be " + "built, how the architecture works, and the acceptance criteria " + "for completion. " + "Compare against the user's original intent in user-intent.md. " + "On REVISE: append your feedback to " + ".factory/strategy/user-intent.md " + "under a new '## [timestamp] Feedback at Strategy Gate' heading " + "before relooping to strategy_director." + ), + ) + + wf.nodes["gate_qa"] = GateNode( + id="gate_qa", + evaluator_type="agent", + evaluator_role=AgentRole.CEO, + gate_prompt=GATE_QA_PROMPT, + reads={ + ".factory/strategy/user-intent.md", + ".factory/reviews/qa-synthesized.md", + }, + ) + + # ── Fix inherited node reads for design-v2 data flow ── + # These nodes inherited reads of adversarial-qa.md from build_workflow, + # but design-v2 replaces that with qa-synthesized.md. + for nid in ("gate_doc_freshness", "gate_precheck", "archivist_build"): + node = wf.nodes[nid] + new_reads = (node.reads - {".factory/reviews/adversarial-qa.md"}) | { + ".factory/reviews/qa-synthesized.md" + } + wf.nodes[nid] = node.model_copy(update={"reads": new_reads}) + + # ── Remove old nodes ── + + _removed = { + "fork_research", + "researcher_similar", + "researcher_techstack", + "researcher_pitfalls", + "join_research", + "gate_research", + "strategist", + "adversarial_tester", + } + for nid in _removed: + wf.nodes.pop(nid, None) + + # ── Rebuild edges ── + + wf.edges = [ + e + for e in wf.edges + if e.source not in _removed + and e.target not in _removed + and not (e.source == "join_qa" and e.target == "gate_qa") + ] + + # Bootstrap + study edges + wf.edges.extend( + [ + *s_edges, + Edge( + source="gate_has_factory", + target="graph_update", + condition=VerdictType.PROCEED, + ), + Edge( + source="gate_has_factory", + target="discover", + condition=VerdictType.HALT, + ), + Edge(source="discover", target="gate_factory_md_exists"), + Edge( + source="gate_factory_md_exists", + target="factory_init", + condition=VerdictType.PROCEED, + ), + Edge( + source="gate_factory_md_exists", + target="create_factory_md", + condition=VerdictType.HALT, + ), + Edge(source="create_factory_md", target="factory_init"), + Edge(source="factory_init", target="graph_update"), + ] + ) + + # Design-v2 specific edges + wf.edges.extend( + [ + Edge(source="init_user_intent", target="gate_has_factory"), + Edge(source="concat_study", target="research_director"), + Edge(source="research_director", target="strategy_director"), + Edge(source="strategy_director", target="synthesize_strategy"), + Edge(source="synthesize_strategy", target="design_doc"), + Edge(source="design_doc", target="gate_strategy"), + Edge( + source="gate_strategy", + target="strategy_director", + condition=VerdictType.RELOOP, + ), + Edge(source="join_qa", target="synthesize_qa"), + Edge(source="synthesize_qa", target="gate_qa"), + ] + ) + + # ── Set workflow metadata ── + + wf.start_node = "init_user_intent" + wf.name = "design-v2" + wf.terminal = True + + return wf diff --git a/factory/workflow/definitions.py b/factory/workflow/definitions.py index 9dd91532d..2a76b67ed 100644 --- a/factory/workflow/definitions.py +++ b/factory/workflow/definitions.py @@ -1384,6 +1384,9 @@ def _get_builtin_registry() -> dict[str, Any]: "outer-loop": lambda: __import__( "factory.workflow.contributed.outer_loop", fromlist=["workflow"] ).workflow(), + "design-v2": lambda: __import__( + "factory.workflow.contributed.design_v2", fromlist=["workflow"] + ).workflow(), } return _BUILTIN_REGISTRY diff --git a/tests/test_workflow_design_v2.py b/tests/test_workflow_design_v2.py new file mode 100644 index 000000000..1b7892001 --- /dev/null +++ b/tests/test_workflow_design_v2.py @@ -0,0 +1,274 @@ +"""Tests for the design-v2 workflow — inference-time scaling with dynamic research/strategy/QA.""" + +from __future__ import annotations + +import pytest + +from factory.workflow.contributed.design_v2 import meta as design_v2_meta +from factory.workflow.contributed.design_v2 import workflow as design_v2_workflow +from factory.workflow.primitives import ( + AgentNode, + AgentRole, + FnNode, + ForkNode, + GateNode, + JoinNode, + VerdictType, +) + + +@pytest.fixture(scope="module") +def design_v2_module(): + class _Module: + meta = design_v2_meta + workflow = staticmethod(design_v2_workflow) + return _Module() + + +@pytest.fixture(scope="module") +def design_v2_wf(): + return design_v2_workflow() + + +# ── Module-level metadata ────────────────────────────────────── + + +class TestMeta: + def test_meta_has_name(self, design_v2_module) -> None: + assert "name" in design_v2_module.meta + assert design_v2_module.meta["name"] == "design-v2" + + def test_meta_has_description(self, design_v2_module) -> None: + assert "description" in design_v2_module.meta + assert len(design_v2_module.meta["description"]) > 0 + + +# ── Graph structure ──────────────────────────────────────────── + + +class TestGraphStructure: + def test_node_count(self, design_v2_wf) -> None: + assert len(design_v2_wf.nodes) == 29 + + def test_edge_count(self, design_v2_wf) -> None: + assert len(design_v2_wf.edges) == 32 + + def test_workflow_name(self, design_v2_wf) -> None: + assert design_v2_wf.name == "design-v2" + + def test_start_node(self, design_v2_wf) -> None: + assert design_v2_wf.start_node == "init_user_intent" + + def test_terminal(self, design_v2_wf) -> None: + assert design_v2_wf.terminal is True + + def test_validates(self, design_v2_wf) -> None: + issues = design_v2_wf.validate_graph() + assert issues == [], f"design-v2 workflow has issues: {issues}" + + +# ── Key nodes present ────────────────────────────────────────── + + +class TestKeyNodesPresent: + @pytest.mark.parametrize( + "node_id", + [ + "init_user_intent", + "research_director", + "strategy_director", + "synthesize_strategy", + "design_doc", + "qa_director", + "synthesize_qa", + "gate_strategy", + "gate_qa", + "fork_qa", + "join_qa", + "health_checker", + "code_reviewer", + "builder", + "gate_has_factory", + "discover", + "graph_update", + "study", + "graph_explorer", + "concat_study", + ], + ) + def test_node_exists(self, design_v2_wf, node_id: str) -> None: + assert node_id in design_v2_wf.nodes, f"missing node: {node_id}" + + +# ── Removed nodes absent ────────────────────────────────────── + + +class TestRemovedNodesAbsent: + @pytest.mark.parametrize( + "node_id", + [ + "fork_research", + "researcher_similar", + "researcher_techstack", + "researcher_pitfalls", + "join_research", + "gate_research", + "strategist", + "adversarial_tester", + ], + ) + def test_node_removed(self, design_v2_wf, node_id: str) -> None: + assert node_id not in design_v2_wf.nodes, f"node should be removed: {node_id}" + + +# ── Node types and properties ───────────────────────────────── + + +class TestNodeProperties: + def test_gate_strategy_is_user(self, design_v2_wf) -> None: + gate = design_v2_wf.nodes["gate_strategy"] + assert isinstance(gate, GateNode) + assert gate.evaluator_type == "user" + + def test_gate_qa_is_agent(self, design_v2_wf) -> None: + gate = design_v2_wf.nodes["gate_qa"] + assert isinstance(gate, GateNode) + assert gate.evaluator_type == "agent" + assert gate.evaluator_role == AgentRole.CEO + + def test_research_director_is_ceo(self, design_v2_wf) -> None: + node = design_v2_wf.nodes["research_director"] + assert isinstance(node, AgentNode) + assert node.role == AgentRole.CEO + assert node.timeout == 3600 + + def test_strategy_director_is_ceo(self, design_v2_wf) -> None: + node = design_v2_wf.nodes["strategy_director"] + assert isinstance(node, AgentNode) + assert node.role == AgentRole.CEO + assert node.timeout == 3600 + + def test_qa_director_is_ceo(self, design_v2_wf) -> None: + node = design_v2_wf.nodes["qa_director"] + assert isinstance(node, AgentNode) + assert node.role == AgentRole.CEO + assert node.timeout == 3600 + + def test_synthesize_strategy_is_strategist(self, design_v2_wf) -> None: + node = design_v2_wf.nodes["synthesize_strategy"] + assert isinstance(node, AgentNode) + assert node.role == AgentRole.STRATEGIST + + def test_design_doc_is_strategist(self, design_v2_wf) -> None: + node = design_v2_wf.nodes["design_doc"] + assert isinstance(node, AgentNode) + assert node.role == AgentRole.STRATEGIST + + def test_synthesize_qa_is_fn_node(self, design_v2_wf) -> None: + node = design_v2_wf.nodes["synthesize_qa"] + assert isinstance(node, FnNode) + assert ".factory/reviews/qa-synthesized.md" in node.writes + + def test_init_user_intent_is_fn_node(self, design_v2_wf) -> None: + node = design_v2_wf.nodes["init_user_intent"] + assert isinstance(node, FnNode) + assert ".factory/strategy/user-intent.md" in node.writes + + def test_fork_qa_targets(self, design_v2_wf) -> None: + fork = design_v2_wf.nodes["fork_qa"] + assert isinstance(fork, ForkNode) + assert set(fork.targets) == {"health_checker", "code_reviewer", "qa_director"} + + def test_join_qa_sources(self, design_v2_wf) -> None: + join = design_v2_wf.nodes["join_qa"] + assert isinstance(join, JoinNode) + assert set(join.sources) == {"health_checker", "code_reviewer", "qa_director"} + + +# ── Edge wiring ──────────────────────────────────────────────── + + +class TestEdgeWiring: + def _edge_set(self, wf): + return {(e.source, e.target, e.condition) for e in wf.edges} + + def test_concat_study_to_research_director(self, design_v2_wf) -> None: + assert ("concat_study", "research_director", None) in self._edge_set(design_v2_wf) + + def test_research_director_to_strategy_director(self, design_v2_wf) -> None: + assert ("research_director", "strategy_director", None) in self._edge_set(design_v2_wf) + + def test_strategy_director_to_synthesize_strategy(self, design_v2_wf) -> None: + assert ("strategy_director", "synthesize_strategy", None) in self._edge_set(design_v2_wf) + + def test_synthesize_strategy_to_design_doc(self, design_v2_wf) -> None: + assert ("synthesize_strategy", "design_doc", None) in self._edge_set(design_v2_wf) + + def test_design_doc_to_gate_strategy(self, design_v2_wf) -> None: + assert ("design_doc", "gate_strategy", None) in self._edge_set(design_v2_wf) + + def test_gate_strategy_reloop_to_strategy_director(self, design_v2_wf) -> None: + assert ( + "gate_strategy", + "strategy_director", + VerdictType.RELOOP, + ) in self._edge_set(design_v2_wf) + + def test_join_qa_to_synthesize_qa(self, design_v2_wf) -> None: + assert ("join_qa", "synthesize_qa", None) in self._edge_set(design_v2_wf) + + def test_synthesize_qa_to_gate_qa(self, design_v2_wf) -> None: + assert ("synthesize_qa", "gate_qa", None) in self._edge_set(design_v2_wf) + + def test_init_user_intent_to_gate_has_factory(self, design_v2_wf) -> None: + assert ("init_user_intent", "gate_has_factory", None) in self._edge_set(design_v2_wf) + + def test_gate_has_factory_routes(self, design_v2_wf) -> None: + edges = self._edge_set(design_v2_wf) + assert ("gate_has_factory", "graph_update", VerdictType.PROCEED) in edges + assert ("gate_has_factory", "discover", VerdictType.HALT) in edges + + def test_no_old_join_qa_to_gate_qa_edge(self, design_v2_wf) -> None: + """The old direct join_qa -> gate_qa edge must be replaced by join_qa -> synthesize_qa.""" + direct = [ + e + for e in design_v2_wf.edges + if e.source == "join_qa" and e.target == "gate_qa" + ] + assert direct == [], "old direct join_qa -> gate_qa edge should be removed" + + +# ── Post checks on director nodes ───────────────────────────── + + +class TestPostChecks: + def test_research_director_post_check(self, design_v2_wf) -> None: + node = design_v2_wf.nodes["research_director"] + assert len(node.post_checks) == 1 + assert node.post_checks[0].path == ".factory/strategy/research-plan.json" + + def test_strategy_director_post_check(self, design_v2_wf) -> None: + node = design_v2_wf.nodes["strategy_director"] + assert len(node.post_checks) == 1 + assert node.post_checks[0].path == ".factory/strategy/strategy-plan.json" + + def test_synthesize_strategy_post_check(self, design_v2_wf) -> None: + node = design_v2_wf.nodes["synthesize_strategy"] + checks = node.post_checks + assert len(checks) == 1 + assert checks[0].path == ".factory/strategy/current.md" + assert checks[0].min_size == 200 + + def test_design_doc_post_check(self, design_v2_wf) -> None: + node = design_v2_wf.nodes["design_doc"] + checks = node.post_checks + assert len(checks) == 1 + assert checks[0].path == ".factory/strategy/current.md" + assert checks[0].min_size == 500 + assert "## What We're Building" in checks[0].must_contain + assert "## Architecture" in checks[0].must_contain + + def test_qa_director_post_check(self, design_v2_wf) -> None: + node = design_v2_wf.nodes["qa_director"] + assert len(node.post_checks) == 1 + assert node.post_checks[0].path == ".factory/reviews/qa-plan.json" diff --git a/workflow-design-v2/SKILL.annotations.yaml b/workflow-design-v2/SKILL.annotations.yaml new file mode 100644 index 000000000..778d9fcf6 --- /dev/null +++ b/workflow-design-v2/SKILL.annotations.yaml @@ -0,0 +1,663 @@ +init_user_intent: + type: FnNode + id: init_user_intent + command: python3 -c "import datetime, os; project = '{project_path}'; ts = datetime.datetime.now().isoformat(timespec='seconds'); + idea = os.environ.get('FOCUS', os.environ.get('FACTORY_IDEA', 'No idea provided')); + content = f'# User Intent Ledger\n\n## [{ts}] Initial Idea\n{idea}\n'; open(f'{project}/.factory/strategy/user-intent.md', + 'w').write(content); print(f'User intent ledger initialized at {ts}')" + reads: [] + writes: + - .factory/strategy/user-intent.md + edges_out: + - target: gate_has_factory + condition: null +gate_has_factory: + type: GateNode + id: gate_has_factory + evaluator_type: fn + evaluator_command: python3 -c "from pathlib import Path; exists = Path("{project_path}/.factory/config.json").exists(); + print("PROCEED" if exists else "HALT")" + reads: [] + edges_out: + - target: graph_update + condition: PROCEED + - target: discover + condition: HALT +discover: + type: FnNode + id: discover + command: factory discover {project_path} + reads: [] + writes: + - .factory/eval_profile.json + edges_out: + - target: gate_factory_md_exists + condition: null +gate_factory_md_exists: + type: GateNode + id: gate_factory_md_exists + evaluator_type: fn + evaluator_command: python3 -c "from pathlib import Path; exists = Path("{project_path}/factory.md").exists(); + print("PROCEED" if exists else "HALT")" + reads: [] + edges_out: + - target: factory_init + condition: PROCEED + - target: create_factory_md + condition: HALT +create_factory_md: + type: AgentNode + id: create_factory_md + role: ceo + blocking: 'true' + reads: + - .factory/eval_profile.json + writes: + - factory.md + edges_out: + - target: factory_init + condition: null + slots: + task_prompt_create_factory_md: 'Create factory.md from template. Copy the factory + config template to the project root. Fill in: Goal, Scope, Guards, Eval command, + Threshold, and Smoke Test. If .factory/eval_spec.json exists, populate the Eval + Spec section. If .factory/strategy/current.md has a Research Configuration section, + populate research sections (Research Target, Mutable/Fixed Surfaces, etc.). + + Read: .factory/eval_profile.json + + Write output to: factory.md' + timeout_create_factory_md: '3600' +factory_init: + type: FnNode + id: factory_init + command: factory init {project_path} + reads: + - factory.md + writes: + - .factory/config.json + edges_out: + - target: graph_update + condition: null +graph_update: + type: FnNode + id: graph_update + command: factory graph update {project_path} + reads: [] + writes: + - graph.json + edges_out: + - target: study + condition: null +study: + type: Study + id: study + command: factory study {project_path} + writes: + - .factory/strategy/observations.md + edges_out: + - target: graph_explorer + condition: null +graph_explorer: + type: AgentNode + id: graph_explorer + role: researcher + blocking: 'true' + reads: + - .factory/strategy/observations.md + writes: + - .factory/strategy/graph-context.md + edges_out: + - target: concat_study + condition: null + slots: + task_prompt_graph_explorer: 'Explore the project''s code knowledge graph to build + structural understanding. Read .factory/strategy/observations.md for focus context. + + + **Step 0 — detect graph availability:** Your working directory is already the + project root. The graph file lives at `$PROJECT_PATH/graph.json` (NOT inside + `.factory/`). Run this smoke check FIRST — use a relative path since your CWD + is the project root: `test -f graph.json && echo ''GRAPH AVAILABLE'' || echo + ''NO GRAPH''` — if the output says GRAPH AVAILABLE, proceed with the graph commands + below. If the output says NO GRAPH, skip to the fallback section. + + + **If the graph IS available:** + + 1. Run `factory graph query "$PROJECT_PATH" "" --depth + 2` to find relevant nodes + + 2. Run `factory graph explain "$PROJECT_PATH" ""` on the most important + nodes to understand their connections and dependencies + + 3. Run `factory graph path "$PROJECT_PATH" "" ""` to trace dependency + paths between key components + + 4. Write structured findings to .factory/strategy/graph-context.md covering: + key modules and their relationships, dependency paths, architectural layers, + entry points and hotspots + + + **If the graph is NOT available**, fall back to direct file exploration: + + 1. Use `find . -name ''*.py'' | head -50` to discover source files + + 2. Use `grep -rn ''class \|def '' --include=''*.py'' | head -100` to map functions + and classes + + 3. Use `grep -rn ''import '' --include=''*.py'' | head -100` to trace dependencies + + 4. Write the same structured findings to .factory/strategy/graph-context.md + + Read: .factory/strategy/observations.md + + Write output to: .factory/strategy/graph-context.md' + timeout_graph_explorer: '600' +concat_study: + type: FnNode + id: concat_study + command: cat {project_path}/.factory/strategy/observations.md {project_path}/.factory/strategy/graph-context.md + > {project_path}/.factory/strategy/study-combined.md + reads: + - .factory/strategy/graph-context.md + - .factory/strategy/observations.md + writes: + - .factory/strategy/study-combined.md + edges_out: + - target: research_director + condition: null +research_director: + type: AgentNode + id: research_director + role: ceo + blocking: 'true' + reads: + - .factory/strategy/study-combined.md + - .factory/strategy/user-intent.md + writes: + - .factory/strategy/research-plan.json + edges_out: + - target: strategy_director + condition: null + slots: + task_prompt_research_director: "You are the Research Director for this design\ + \ session.\n\nRead:\n- `.factory/strategy/study-combined.md` — project study\ + \ findings (observations + graph analysis)\n- `.factory/strategy/user-intent.md`\ + \ — the user's original idea\n\nYour task has TWO phases:\n\nPHASE 1 — DESIGN\ + \ RESEARCH DIRECTIONS\nAnalyze the design space and identify N orthogonal research\ + \ dimensions.\nDefault dimensions (adapt or replace based on the specific project):\n\ + \ - Similar projects / prior art\n - Technology stack options\n - Common\ + \ pitfalls and failure modes\nYou may add dimensions specific to the domain\ + \ (e.g., security, UX patterns,\ndata model constraints, compliance requirements,\ + \ API design, concurrency).\n\nN is NOT fixed — YOU decide based on domain complexity:\n\ + \ - Simple CLI or library: 3 directions\n - Web app with auth, database, UI:\ + \ 4-5 directions\n - Complex system with integrations, security, compliance:\ + \ 5-7 directions\n\nFor each direction, design a TAILORED prompt — not a generic\ + \ template.\nBad: \"Research similar projects and prior art\"\nGood: \"Find\ + \ existing link-checking tools that handle Obsidian-style wikilinks\n \ + \ ([[note]]) and image embeds (![[image.png]]). Compare how they resolve\n \ + \ relative paths vs vault-root-relative paths.\"\n\nWrite the research\ + \ plan to `.factory/strategy/research-plan.json`:\n```json\n[\n {\"focus\"\ + : \"...\", \"slug\": \"...\", \"prompt\": \"...\"}\n]\n```\n\nConstraints:\n\ + - Minimum 3 directions, maximum 7\n- Each slug must be unique and kebab-case\n\ + - Prompts must be specific to THIS project, not generic templates\n\nPHASE 2\ + \ — EXECUTE RESEARCH\nFor each direction in the plan, spawn a researcher agent:\n\ + ```\nfactory agent researcher --task \"\" --project $PROJECT_PATH\n\ + ```\n\nEach researcher writes to `.factory/strategy/research-.md`.\n\n\ + After ALL researchers complete, review quality:\n- Each research file exists\ + \ and has substantive content (>50 bytes)\n- No two reports cover the same ground\ + \ excessively\n- Key risks and opportunities are covered\n\nIf a researcher\ + \ produced thin output, re-invoke it with a more specific prompt.\n\nWrite a\ + \ brief research summary to the end of research-plan.json noting\nwhich directions\ + \ completed and any quality issues.\nRead: .factory/strategy/study-combined.md,\ + \ .factory/strategy/user-intent.md\nWrite output to: .factory/strategy/research-plan.json" + timeout_research_director: '3600' +strategy_director: + type: AgentNode + id: strategy_director + role: ceo + blocking: 'true' + reads: + - .factory/strategy/research-plan.json + - .factory/strategy/study-combined.md + - .factory/strategy/user-intent.md + writes: + - .factory/strategy/strategy-plan.json + edges_out: + - target: synthesize_strategy + condition: null + slots: + task_prompt_strategy_director: "You are the Strategy Director for this design\ + \ session.\n\nRead:\n- ALL research reports at `.factory/strategy/research-*.md`\n\ + - `.factory/strategy/research-plan.json` — which research directions were explored\n\ + - `.factory/strategy/user-intent.md` — the user's original idea\n- `.factory/strategy/study-combined.md`\ + \ — project context\n\nYour task has TWO phases:\n\nPHASE 1 — DESIGN STRATEGY\ + \ PERSPECTIVES\nAnalyze the research findings and user intent to identify M\ + \ strategy\nperspectives this project needs.\n\nDefault perspectives (adapt\ + \ or replace based on the specific project):\n - Architecture strategy — how\ + \ to build it (components, phases, tech choices)\n - Testing/verification strategy\ + \ — how to verify it works (acceptance criteria,\n test plan, edge cases)\n\ + \ - Risk/scope strategy — what to cut, what's hard, what breaks\n\nYou may\ + \ add perspectives specific to the domain:\n - Security strategy (for auth-heavy\ + \ projects)\n - Data modeling strategy (for data-heavy projects)\n - API design\ + \ strategy (for API-first projects)\n - Performance strategy (for latency-sensitive\ + \ systems)\n\nM is NOT fixed — YOU decide based on project complexity:\n -\ + \ Simple project: 2-3 perspectives\n - Medium project: 3-4 perspectives\n \ + \ - Complex project: 4-5 perspectives\n\nFor each perspective, design a TAILORED\ + \ prompt.\nBad: \"Create an architecture strategy\"\nGood: \"Design the architecture\ + \ for a markdown link checker CLI. The core\n challenge is resolving Obsidian\ + \ wikilinks against a configurable vault\n root while also supporting\ + \ standard URLs with redirect following.\n Research shows \ + \ handles HTTP well but nothing handles\n wikilinks — design that component\ + \ from scratch.\"\n\nWrite the strategy plan to `.factory/strategy/strategy-plan.json`:\n\ + ```json\n[\n {\"perspective\": \"...\", \"slug\": \"...\", \"prompt\": \"...\"\ + }\n]\n```\n\nConstraints:\n- Minimum 2 perspectives, maximum 5\n- Each slug\ + \ must be unique and kebab-case\n- One perspective MUST cover testing/verification\ + \ with explicit acceptance criteria\n- Prompts must reference specific findings\ + \ from the research reports\n\nPHASE 2 — EXECUTE STRATEGIES\nFor each perspective\ + \ in the plan, spawn a strategist agent:\n```\nfactory agent strategist --task\ + \ \"\" --project $PROJECT_PATH\n```\n\nEach strategist writes\ + \ to `.factory/strategy/strategy-.md`.\n\nAfter ALL strategists complete,\ + \ review quality:\n- Each strategy file exists and has substantive content (>100\ + \ bytes)\n- The testing strategy has a `### Acceptance Criteria` section with\ + \ checkboxes\n- Architecture strategy cites research findings\n- No critical\ + \ perspective is missing\n\nIf a strategist produced thin output, re-invoke\ + \ it with a more specific prompt.\n\nWrite a brief strategy summary to the end\ + \ of strategy-plan.json noting\nwhich perspectives completed and any quality\ + \ issues.\nRead: .factory/strategy/research-plan.json, .factory/strategy/study-combined.md,\ + \ .factory/strategy/user-intent.md\nWrite output to: .factory/strategy/strategy-plan.json" + timeout_strategy_director: '3600' +synthesize_strategy: + type: AgentNode + id: synthesize_strategy + role: strategist + blocking: 'true' + reads: + - .factory/strategy/user-intent.md + writes: + - .factory/strategy/current.md + edges_out: + - target: design_doc + condition: null + slots: + task_prompt_synthesize_strategy: "You are the Strategy Synthesizer. Compile one\ + \ final plan from all\nstrategy inputs.\n\nRead:\n- ALL strategy files at `.factory/strategy/strategy-*.md`\n\ + - `.factory/strategy/strategy-plan.json` — which perspectives were explored\n\ + - `.factory/strategy/user-intent.md` — ground truth for user's ask\n\nWrite\ + \ the final plan to `.factory/strategy/current.md`.\n\nRequired sections (in\ + \ this order):\n### Architecture\n Merge from architecture strategy. Include\ + \ component list and interfaces.\n### Phased Plan\n #### Phase 1: \n\ + \ - **What:** \n - **Why:** \n\ + \ - **Acceptance criteria:** \n\ + \ - **Risks:** \n #### Phase 2:\ + \ \n ...\n### Acceptance Criteria\n Full checklist from testing strategy.\ + \ Each item must be:\n - [ ] Specific enough for pass/fail verification\n \ + \ - Traceable to user intent (cite which part of user-intent.md)\n### MVP Scope\n\ + \ From risk strategy. In vs deferred.\n### Deferred Features\n Items requiring\ + \ human intervention or explicitly deferred.\n\nCRITICAL: The ### Acceptance\ + \ Criteria section is the contract between\nthe builder and QA. It flows to\ + \ adversarial testers who verify each\ncriterion independently. Make every item\ + \ testable and unambiguous.\nRead: .factory/strategy/user-intent.md\nWrite output\ + \ to: .factory/strategy/current.md" + timeout_synthesize_strategy: '600' +design_doc: + type: AgentNode + id: design_doc + role: strategist + blocking: 'true' + reads: + - .factory/strategy/current.md + - .factory/strategy/user-intent.md + writes: + - .factory/strategy/current.md + edges_out: + - target: gate_strategy + condition: null + slots: + task_prompt_design_doc: "You are a Technical Writer and Design Architect. Your\ + \ job: take the structured\nstrategy at `.factory/strategy/current.md` and rewrite\ + \ it as a proper\nDESIGN DOCUMENT — a document a human reviewer can read end-to-end\ + \ and\nunderstand exactly what is being built, why, and how.\n\nRead:\n- `.factory/strategy/current.md`\ + \ — the structured strategy (your raw input)\n- `.factory/strategy/user-intent.md`\ + \ — the user's original idea and feedback\n\nRewrite `.factory/strategy/current.md`\ + \ IN PLACE. Replace the compressed\nbullet points with a well-structured design\ + \ document.\n\nRequired sections (in this order):\n\n## What We're Building\n\ + \ Explain the project in 2-3 paragraphs of prose. What does it do?\n Who is\ + \ it for? What problem does it solve? Reference the user's\n original words\ + \ from user-intent.md.\n\n## Architecture\n Describe the system architecture\ + \ in full sentences.\n Include a text-based architecture diagram using box-drawing\ + \ characters:\n ```\n ┌──────────┐ ┌──────────┐ ┌──────────┐\n │\ + \ Component│────▶│ Component│────▶│ Component│\n └──────────┘ └──────────┘\ + \ └──────────┘\n ```\n Explain each component — what it does, why it exists,\ + \ how it connects.\n\n## How It Works\n Walk through the user flow step by\ + \ step. For a CLI, show example\n invocations and expected output. For a web\ + \ app, describe the user\n journey screen by screen. For a library, show usage\ + \ examples.\n\n This should read like a tutorial — someone unfamiliar with\ + \ the project\n should be able to follow along.\n\n## Phased Plan\n For each\ + \ phase, explain:\n - What gets built in this phase and why this ordering\n\ + \ - What the user can do after this phase completes\n - How to verify the\ + \ phase worked (concrete test commands or checks)\n\n Use prose paragraphs,\ + \ not just bullet points. Each phase should\n read as a self-contained \"chapter.\"\ + \n\n## Acceptance Criteria\n Present the full checklist, but group criteria\ + \ by category and add\n context for each. Explain WHY each criterion matters,\ + \ not just what it is.\n\n Format: category heading, then checkbox items with\ + \ brief explanation.\n\n## MVP Scope\n What's in, what's deferred, and why.\ + \ Explain the tradeoffs.\n\n## Deferred Features\n Items deferred to future\ + \ phases, with brief rationale.\n\nCRITICAL RULES:\n- Write in full sentences\ + \ and paragraphs, NOT bullet points\n- The reader should understand the design\ + \ WITHOUT reading any other file\n- Include concrete examples (CLI invocations,\ + \ API calls, code snippets)\n- Architecture diagrams must use text/box-drawing\ + \ characters\n- Every technical choice must be explained — no unexplained jargon\n\ + - The document must be self-contained: a human reviewer reads ONLY this\n file\ + \ and decides whether to approve the design\nRead: .factory/strategy/current.md,\ + \ .factory/strategy/user-intent.md\nWrite output to: .factory/strategy/current.md" + timeout_design_doc: '600' +gate_strategy: + type: GateNode + id: gate_strategy + evaluator_type: user + reads: + - .factory/strategy/current.md + - .factory/strategy/user-intent.md + edges_out: + - target: archivist_plan + condition: PROCEED + - target: strategy_director + condition: RELOOP + slots: + max_iterations_gate_strategy: '3' +archivist_plan: + type: AgentNode + id: archivist_plan + role: archivist + blocking: 'false' + reads: + - .factory/strategy/current.md + writes: + - .factory/archive/plan.md + edges_out: + - target: builder + condition: null + slots: + task_prompt_archivist_plan: 'Archive the approved research and strategy. + + Read: .factory/strategy/current.md + + Write output to: .factory/archive/plan.md' + timeout_archivist_plan: '300' +builder: + type: AgentNode + id: builder + role: builder + blocking: 'true' + reads: + - .factory/strategy/current.md + writes: + - .factory/reviews/builder-latest.md + edges_out: + - target: gate_build + condition: null + slots: + task_prompt_builder: 'Implement the next phase from .factory/strategy/current.md. + Read the CEO''s plan approval at .factory/reviews/ceo-verdict-strategist.md. + Read CLAUDE.md and factory.md if they exist. Implement exactly what the current + phase describes. Run tests. Commit changes and open a draft PR. + + Read: .factory/strategy/current.md + + Write output to: .factory/reviews/builder-latest.md' + timeout_builder: '1200' +gate_build: + type: GateNode + id: gate_build + evaluator_type: agent + evaluator_role: ceo + reads: + - .factory/reviews/builder-latest.md + edges_out: + - target: fork_qa + condition: PROCEED + - target: builder + condition: RELOOP + slots: + gate_prompt_gate_build: Read builder output. Check git log and diff. Does the + work match the plan for this phase? If the Builder opened a PR, read it. REDIRECT + if off-scope or missed key requirements. + max_iterations_gate_build: '3' +fork_qa: + type: ForkNode + id: fork_qa + targets: health_checker,code_reviewer,qa_director + edges_out: + - target: join_qa + condition: null +health_checker: + type: AgentNode + id: health_checker + role: health_checker + blocking: 'true' + reads: + - .factory/reviews/builder-latest.md + - .factory/strategy/current.md + writes: + - .factory/reviews/health-check.md + edges_out: [] + slots: + task_prompt_health_checker: 'Execute health_checker task for the project. + + Read: .factory/reviews/builder-latest.md, .factory/strategy/current.md + + Write output to: .factory/reviews/health-check.md' + timeout_health_checker: '600' +code_reviewer: + type: AgentNode + id: code_reviewer + role: code_reviewer + blocking: 'true' + reads: + - .factory/reviews/builder-latest.md + - .factory/strategy/current.md + writes: + - .factory/reviews/code-review.md + edges_out: [] + slots: + task_prompt_code_reviewer: 'Execute code_reviewer task for the project. + + Read: .factory/reviews/builder-latest.md, .factory/strategy/current.md + + Write output to: .factory/reviews/code-review.md' + timeout_code_reviewer: '900' +qa_director: + type: AgentNode + id: qa_director + role: ceo + blocking: 'true' + reads: + - .factory/reviews/builder-latest.md + - .factory/strategy/current.md + - .factory/strategy/user-intent.md + writes: + - .factory/reviews/qa-plan.json + edges_out: [] + slots: + task_prompt_qa_director: "You are the QA Director for this design session.\n\n\ + Read:\n- `.factory/strategy/current.md` — the design document with acceptance\ + \ criteria\n- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked\ + \ for\n- `.factory/reviews/builder-latest.md` — what the builder implemented\n\ + \nYour task has TWO phases:\n\nPHASE 1 — DESIGN QA APPROACHES\nAnalyze the acceptance\ + \ criteria, the user's intent, and the builder's\nimplementation to identify\ + \ K orthogonal testing approaches.\n\nDefault approaches (adapt or replace based\ + \ on the specific project):\n - Happy path verification — does each acceptance\ + \ criterion pass with\n normal, expected inputs?\n - Edge case & boundary\ + \ testing — empty inputs, large inputs, special\n characters, off-by-one,\ + \ boundary conditions\n - User intent verification — does the output match\ + \ what the user\n ACTUALLY asked for (from user-intent.md), not just the\ + \ plan?\n\nYou may add approaches specific to the implementation:\n - Security\ + \ testing (for auth, file I/O, user-facing APIs)\n - Integration testing (for\ + \ multi-component systems)\n - Performance testing (for latency-sensitive features)\n\ + \ - Error handling testing (for systems with many failure modes)\n - Concurrency\ + \ testing (for parallel/async systems)\n\nK is NOT fixed — YOU decide based\ + \ on the complexity of the acceptance\ncriteria and the nature of the implementation:\n\ + \ - Simple implementation (3-5 criteria): 2 testers\n - Medium implementation\ + \ (5-10 criteria): 3 testers\n - Complex implementation (10+ criteria, security,\ + \ integrations): 4-5 testers\n\nFor each approach, design a TAILORED prompt\ + \ — not a generic template.\nBad: \"Test the implementation for edge cases\"\ + \nGood: \"Test the Obsidian wikilink resolver with these edge cases:\n \ + \ nested vault folders (vault/sub/note.md linking to ../other.md),\n \ + \ wikilinks with display text ([[note|display]]),\n wikilinks to non-existent\ + \ files, wikilinks with anchor\n fragments ([[note#heading]]), and case-sensitivity\ + \ mismatches.\"\n\nWrite the QA plan to `.factory/reviews/qa-plan.json`:\n```json\n\ + [\n {\"approach\": \"...\", \"slug\": \"...\", \"prompt\": \"...\"}\n]\n```\n\ + \nConstraints:\n- Minimum 2 approaches, maximum 5\n- Each slug must be unique\ + \ and kebab-case\n- ALL acceptance criteria from current.md must be covered\ + \ by at least\n one tester's prompt\n- One approach MUST verify user intent\ + \ against user-intent.md\n- Prompts must reference specific acceptance criteria\ + \ and implementation details\n\nPHASE 2 — EXECUTE QA\nFor each approach in the\ + \ plan, spawn an adversarial tester agent:\n```\nfactory agent adversarial_tester\ + \ --task \"\" --project $PROJECT_PATH\n```\n\nEach tester writes\ + \ to `.factory/reviews/adversarial--latest.md`.\n\nAfter ALL testers complete,\ + \ review quality:\n- Each adversarial report exists and has substantive findings\n\ + - All acceptance criteria are covered by at least one tester\n- No tester missed\ + \ its assigned focus area\n- Critical findings are actually reproducible (spot-check)\n\ + \nIf a tester produced thin output, re-invoke it with a more specific prompt.\n\ + \nWrite a brief QA summary to the end of qa-plan.json noting which\napproaches\ + \ completed and any quality issues.\nRead: .factory/reviews/builder-latest.md,\ + \ .factory/strategy/current.md, .factory/strategy/user-intent.md\nWrite output\ + \ to: .factory/reviews/qa-plan.json" + timeout_qa_director: '3600' +join_qa: + type: JoinNode + id: join_qa + sources: health_checker,code_reviewer,qa_director + reads: + - .factory/reviews/code-review.md + - .factory/reviews/health-check.md + - .factory/reviews/qa-plan.json + writes: [] + edges_out: + - target: synthesize_qa + condition: null +synthesize_qa: + type: FnNode + id: synthesize_qa + command: 'python3 -c "from pathlib import Path; import re, glob; project = ''{project_path}''; + reports = []; for p in sorted(Path(f''{project}/.factory/reviews'').glob(''adversarial-*-latest.md'')): slug + = p.name.replace(''-latest.md'', '''').replace(''adversarial-'', ''''); reports.append((slug, + p.read_text())); findings = {}; for tester_slug, text in reports: for line + in text.splitlines(): stripped = line.strip(); if stripped.startswith(''- + '') or stripped.startswith(''* ''): key = re.sub(r''\s+'', '' '', + stripped[2:].strip().lower()[:80]); findings.setdefault(key, []).append(tester_slug); + high = [(k, v) for k, v in findings.items() if len(v) >= 2]; medium = [(k, v) + for k, v in findings.items() if len(v) == 1]; out = [''# Synthesized QA Report\n'']; + hc = Path(f''{project}/.factory/reviews/health-check.md''); cr = Path(f''{project}/.factory/reviews/code-review.md''); + out.append(''## Health Check\n''); out.append(hc.read_text() if hc.exists() else + ''(not available)''); out.append(''\n## Code Review\n''); out.append(cr.read_text() + if cr.exists() else ''(not available)''); out.append(''\n## High-Confidence Adversarial + Findings (caught by 2+ testers)\n''); [out.append(f''- {k} (testers: {v})'') for + k, v in high]; if not high: out.append(''- (none)''); out.append(''\n## Medium-Confidence + Adversarial Findings (single tester)\n''); [out.append(f''- {k} (tester: {v[0]})'') + for k, v in medium]; if not medium: out.append(''- (none)''); out.append(''\n## + Raw Adversarial Reports\n''); [out.append(f''### Tester: {slug}\n{text}\n'') for + slug, text in reports]; Path(f''{project}/.factory/reviews/qa-synthesized.md'').write_text(''\n''.join(out)); + print(f''Synthesized {len(high)} high + {len(medium)} medium findings from {len(reports)} + adversarial reports'')"' + reads: + - .factory/reviews/code-review.md + - .factory/reviews/health-check.md + writes: + - .factory/reviews/qa-synthesized.md + edges_out: + - target: gate_qa + condition: null +gate_qa: + type: GateNode + id: gate_qa + evaluator_type: agent + evaluator_role: ceo + reads: + - .factory/reviews/qa-synthesized.md + - .factory/strategy/user-intent.md + edges_out: + - target: gate_doc_freshness + condition: PROCEED + - target: builder + condition: RELOOP + slots: + gate_prompt_gate_qa: "You are the CEO reviewing QA results. This is the final\ + \ gate before merge.\n\nRead:\n- `.factory/strategy/user-intent.md` — what the\ + \ user ACTUALLY asked for\n- `.factory/reviews/qa-synthesized.md` — merged QA\ + \ report (health check,\n code review, and synthesized adversarial findings)\n\ + \nDecision framework:\n\nPROCEED if ALL of these hold:\n 1. All acceptance\ + \ criteria from current.md are verified PASS\n 2. Health check passes (tests\ + \ green, eval not regressed)\n 3. No blocking code review issues\n 4. No HIGH-confidence\ + \ adversarial findings that violate user intent\n\nRELOOP to builder (max 3\ + \ iterations) if ANY of these hold:\n 1. An acceptance criterion failed — cite\ + \ which one\n 2. Health check failed — cite which check\n 3. Blocking review\ + \ or HIGH adversarial findings\n\n When relooping, provide feedback mapped\ + \ to SPECIFIC user requirements:\n - \"User asked for X (user-intent.md), but\ + \ \"\n - \"Acceptance criterion '' FAILED: \"\ + \n\n IMPORTANT: Append your reloop feedback to .factory/strategy/user-intent.md\n\ + \ under a new '## [timestamp] Reloop Feedback (Iteration N)' heading.\n\nHALT\ + \ if:\n - 3 reloops exhausted without resolution\n - Fundamental design flaw\ + \ that builder iterations cannot fix" + max_iterations_gate_qa: '3' +gate_doc_freshness: + type: GateNode + id: gate_doc_freshness + evaluator_type: agent + evaluator_role: ceo + reads: + - .factory/reviews/qa-synthesized.md + edges_out: + - target: gate_precheck + condition: PROCEED + - target: builder + condition: RELOOP + slots: + gate_prompt_gate_doc_freshness: Check the PR diff for documentation freshness. + If public APIs, CLI commands, configuration options, or architecture were changed + or added, corresponding documentation (README.md, CLAUDE.md, docstrings, --help + text, or doc/ files) MUST be updated. PROCEED if docs are current or no doc-worthy + changes exist. RELOOP to builder if documentation is stale — specify exactly + which changes need doc updates. + max_iterations_gate_doc_freshness: '3' +gate_precheck: + type: GateNode + id: gate_precheck + evaluator_type: fn + evaluator_command: factory precheck {project_path} --score-before 0 --score-after + 0 + reads: + - .factory/reviews/qa-synthesized.md + edges_out: + - target: archivist_build + condition: PROCEED + - target: archivist_build + condition: HALT +archivist_build: + type: AgentNode + id: archivist_build + role: archivist + blocking: 'false' + reads: + - .factory/reviews/qa-synthesized.md + writes: + - .factory/archive/build.md + edges_out: + - target: spec_generate + condition: null + slots: + task_prompt_archivist_build: 'Archive the build phase results. + + Read: .factory/reviews/qa-synthesized.md + + Write output to: .factory/archive/build.md' + timeout_archivist_build: '300' +spec_generate: + type: FnNode + id: spec_generate + command: factory workflow run spec-generate {project_path} + reads: [] + writes: [] + edges_out: [] diff --git a/workflow-design-v2/SKILL.md b/workflow-design-v2/SKILL.md new file mode 100644 index 000000000..a63b51264 --- /dev/null +++ b/workflow-design-v2/SKILL.md @@ -0,0 +1,731 @@ +--- +name: workflow-design-v2 +description: "Run the design-v2 workflow." +disable-model-invocation: true +argument-hint: "" +--- + +# Design V2 Workflow + +The user wants: **$ARGUMENTS** + +## Step: Init User Intent + +Creates the user intent ledger with the initial idea. + +```bash +python3 -c "import datetime, os; project = '$PROJECT_PATH'; ts = datetime.datetime.now().isoformat(timespec='seconds'); idea = os.environ.get('FOCUS', os.environ.get('FACTORY_IDEA', 'No idea provided')); content = f'# User Intent Ledger\n\n## [{ts}] Initial Idea\n{idea}\n'; open(f'{project}/.factory/strategy/user-intent.md', 'w').write(content); print(f'User intent ledger initialized at {ts}')" +``` + +### Gate — Has Factory (Automated) + +**MANDATORY:** Wait for the preceding agent to finish, then run this check BEFORE spawning the next agent. Do NOT run agents in parallel across this gate. + +```bash +python3 -c "from pathlib import Path; exists = Path("$PROJECT_PATH/.factory/config.json").exists(); print("PROCEED" if exists else "HALT")" +``` + +- **PROCEED** (exit 0 / no FAIL in output) → continue to `graph_update` +- **HALT** (exit non-zero / FAIL in output) → continue to `discover` instead. + +## Step: Discover + +```bash +factory discover $PROJECT_PATH +``` + +### Gate — Factory Md Exists (Automated) + +**MANDATORY:** Wait for the preceding agent to finish, then run this check BEFORE spawning the next agent. Do NOT run agents in parallel across this gate. + +```bash +python3 -c "from pathlib import Path; exists = Path("$PROJECT_PATH/factory.md").exists(); print("PROCEED" if exists else "HALT")" +``` + +- **PROCEED** (exit 0 / no FAIL in output) → continue to `factory_init` +- **HALT** (exit non-zero / FAIL in output) → continue to `create_factory_md` instead. + +## Phase 1: Ceo — Create Factory Md + +```bash +factory agent ceo --task "Create factory.md from template. Copy the factory config template to the project root. Fill in: Goal, Scope, Guards, Eval command, Threshold, and Smoke Test. If .factory/eval_spec.json exists, populate the Eval Spec section. If .factory/strategy/current.md has a Research Configuration section, populate research sections (Research Target, Mutable/Fixed Surfaces, etc.). +Read: .factory/eval_profile.json +Write output to: factory.md" --project "$PROJECT_PATH" --timeout 3600 +``` + +```bash +# Artifact verification: create_factory_md +_vfail=0 +_f="$PROJECT_PATH/factory.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: create_factory_md: factory.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: create_factory_md: factory.md is empty" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=create_factory_md" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: create_factory_md artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=create_factory_md" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" +``` +*(harness verification — DO NOT SKIP)* + +## Step: Factory Init + +Parse factory.md and generate .factory/config.json. Must run after factory.md is created. + +```bash +factory init $PROJECT_PATH +``` + +## Step: Graph Update + +Extract or incrementally update the code knowledge graph before study. + +```bash +factory graph update $PROJECT_PATH +``` + +## Phase 2: Observe + +Run local study to gather observations: + +```bash +factory study $PROJECT_PATH +``` + +Writes observations to `.factory/strategy/observations.md`. + +If your task includes a focus directive or focus topic, pass it to the study command: +`factory study $PROJECT_PATH --focus ""` + +## Phase 3: Researcher — Graph Explorer + +```bash +factory agent researcher --task "Explore the project's code knowledge graph to build structural understanding. Read .factory/strategy/observations.md for focus context. + +**Step 0 — detect graph availability:** Your working directory is already the project root. The graph file lives at `$PROJECT_PATH/graph.json` (NOT inside `.factory/`). Run this smoke check FIRST — use a relative path since your CWD is the project root: `test -f graph.json && echo 'GRAPH AVAILABLE' || echo 'NO GRAPH'` — if the output says GRAPH AVAILABLE, proceed with the graph commands below. If the output says NO GRAPH, skip to the fallback section. + +**If the graph IS available:** +1. Run `factory graph query "$PROJECT_PATH" "" --depth 2` to find relevant nodes +2. Run `factory graph explain "$PROJECT_PATH" ""` on the most important nodes to understand their connections and dependencies +3. Run `factory graph path "$PROJECT_PATH" "" ""` to trace dependency paths between key components +4. Write structured findings to .factory/strategy/graph-context.md covering: key modules and their relationships, dependency paths, architectural layers, entry points and hotspots + +**If the graph is NOT available**, fall back to direct file exploration: +1. Use `find . -name '*.py' | head -50` to discover source files +2. Use `grep -rn 'class \|def ' --include='*.py' | head -100` to map functions and classes +3. Use `grep -rn 'import ' --include='*.py' | head -100` to trace dependencies +4. Write the same structured findings to .factory/strategy/graph-context.md +Read: .factory/strategy/observations.md +Write output to: .factory/strategy/graph-context.md" --project "$PROJECT_PATH" --timeout 600 +``` + +```bash +# Artifact verification: graph_explorer +_vfail=0 +_f="$PROJECT_PATH/.factory/strategy/graph-context.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: graph_explorer: .factory/strategy/graph-context.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: graph_explorer: .factory/strategy/graph-context.md is empty" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=graph_explorer" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: graph_explorer artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=graph_explorer" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" +``` +*(harness verification — DO NOT SKIP)* + +## Step: Concat Study + +```bash +cat $PROJECT_PATH/.factory/strategy/observations.md $PROJECT_PATH/.factory/strategy/graph-context.md > $PROJECT_PATH/.factory/strategy/study-combined.md +``` + +## Phase 4: Ceo — Research Director + +```bash +factory agent ceo --task "You are the Research Director for this design session. + +Read: +- `.factory/strategy/study-combined.md` — project study findings (observations + graph analysis) +- `.factory/strategy/user-intent.md` — the user's original idea + +Your task has TWO phases: + +PHASE 1 — DESIGN RESEARCH DIRECTIONS +Analyze the design space and identify N orthogonal research dimensions. +Default dimensions (adapt or replace based on the specific project): + - Similar projects / prior art + - Technology stack options + - Common pitfalls and failure modes +You may add dimensions specific to the domain (e.g., security, UX patterns, +data model constraints, compliance requirements, API design, concurrency). + +N is NOT fixed — YOU decide based on domain complexity: + - Simple CLI or library: 3 directions + - Web app with auth, database, UI: 4-5 directions + - Complex system with integrations, security, compliance: 5-7 directions + +For each direction, design a TAILORED prompt — not a generic template. +Bad: "Research similar projects and prior art" +Good: "Find existing link-checking tools that handle Obsidian-style wikilinks + ([[note]]) and image embeds (![[image.png]]). Compare how they resolve + relative paths vs vault-root-relative paths." + +Write the research plan to `.factory/strategy/research-plan.json`: +```json +[ + {"focus": "...", "slug": "...", "prompt": "..."} +] +``` + +Constraints: +- Minimum 3 directions, maximum 7 +- Each slug must be unique and kebab-case +- Prompts must be specific to THIS project, not generic templates + +PHASE 2 — EXECUTE RESEARCH +For each direction in the plan, spawn a researcher agent: +``` +factory agent researcher --task "" --project $PROJECT_PATH +``` + +Each researcher writes to `.factory/strategy/research-.md`. + +After ALL researchers complete, review quality: +- Each research file exists and has substantive content (>50 bytes) +- No two reports cover the same ground excessively +- Key risks and opportunities are covered + +If a researcher produced thin output, re-invoke it with a more specific prompt. + +Write a brief research summary to the end of research-plan.json noting +which directions completed and any quality issues. +Read: .factory/strategy/study-combined.md, .factory/strategy/user-intent.md +Write output to: .factory/strategy/research-plan.json" --project "$PROJECT_PATH" --timeout 3600 +``` + +```bash +# Artifact verification: research_director +_vfail=0 +_f="$PROJECT_PATH/.factory/strategy/research-plan.json" +[ ! -f "$_f" ] && echo "VERIFY FAIL: research_director: .factory/strategy/research-plan.json missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: research_director: .factory/strategy/research-plan.json is empty" && _vfail=1 +[ -f "$_f" ] && [ "$(wc -c < "$_f")" -lt 20 ] && echo "VERIFY FAIL: research_director: .factory/strategy/research-plan.json smaller than 20 bytes" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=research_director" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: research_director artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=research_director" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" +``` +*(harness verification — DO NOT SKIP)* + +## Phase 5: Ceo — Strategy Director + +```bash +factory agent ceo --task "You are the Strategy Director for this design session. + +Read: +- ALL research reports at `.factory/strategy/research-*.md` +- `.factory/strategy/research-plan.json` — which research directions were explored +- `.factory/strategy/user-intent.md` — the user's original idea +- `.factory/strategy/study-combined.md` — project context + +Your task has TWO phases: + +PHASE 1 — DESIGN STRATEGY PERSPECTIVES +Analyze the research findings and user intent to identify M strategy +perspectives this project needs. + +Default perspectives (adapt or replace based on the specific project): + - Architecture strategy — how to build it (components, phases, tech choices) + - Testing/verification strategy — how to verify it works (acceptance criteria, + test plan, edge cases) + - Risk/scope strategy — what to cut, what's hard, what breaks + +You may add perspectives specific to the domain: + - Security strategy (for auth-heavy projects) + - Data modeling strategy (for data-heavy projects) + - API design strategy (for API-first projects) + - Performance strategy (for latency-sensitive systems) + +M is NOT fixed — YOU decide based on project complexity: + - Simple project: 2-3 perspectives + - Medium project: 3-4 perspectives + - Complex project: 4-5 perspectives + +For each perspective, design a TAILORED prompt. +Bad: "Create an architecture strategy" +Good: "Design the architecture for a markdown link checker CLI. The core + challenge is resolving Obsidian wikilinks against a configurable vault + root while also supporting standard URLs with redirect following. + Research shows handles HTTP well but nothing handles + wikilinks — design that component from scratch." + +Write the strategy plan to `.factory/strategy/strategy-plan.json`: +```json +[ + {"perspective": "...", "slug": "...", "prompt": "..."} +] +``` + +Constraints: +- Minimum 2 perspectives, maximum 5 +- Each slug must be unique and kebab-case +- One perspective MUST cover testing/verification with explicit acceptance criteria +- Prompts must reference specific findings from the research reports + +PHASE 2 — EXECUTE STRATEGIES +For each perspective in the plan, spawn a strategist agent: +``` +factory agent strategist --task "" --project $PROJECT_PATH +``` + +Each strategist writes to `.factory/strategy/strategy-.md`. + +After ALL strategists complete, review quality: +- Each strategy file exists and has substantive content (>100 bytes) +- The testing strategy has a `### Acceptance Criteria` section with checkboxes +- Architecture strategy cites research findings +- No critical perspective is missing + +If a strategist produced thin output, re-invoke it with a more specific prompt. + +Write a brief strategy summary to the end of strategy-plan.json noting +which perspectives completed and any quality issues. +Read: .factory/strategy/research-plan.json, .factory/strategy/study-combined.md, .factory/strategy/user-intent.md +Write output to: .factory/strategy/strategy-plan.json" --project "$PROJECT_PATH" --timeout 3600 +``` + +```bash +# Artifact verification: strategy_director +_vfail=0 +_f="$PROJECT_PATH/.factory/strategy/strategy-plan.json" +[ ! -f "$_f" ] && echo "VERIFY FAIL: strategy_director: .factory/strategy/strategy-plan.json missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: strategy_director: .factory/strategy/strategy-plan.json is empty" && _vfail=1 +[ -f "$_f" ] && [ "$(wc -c < "$_f")" -lt 20 ] && echo "VERIFY FAIL: strategy_director: .factory/strategy/strategy-plan.json smaller than 20 bytes" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=strategy_director" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: strategy_director artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=strategy_director" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" +``` +*(harness verification — DO NOT SKIP)* + +## Phase 6: Strategist — Synthesize Strategy + +```bash +factory agent strategist --task "You are the Strategy Synthesizer. Compile one final plan from all +strategy inputs. + +Read: +- ALL strategy files at `.factory/strategy/strategy-*.md` +- `.factory/strategy/strategy-plan.json` — which perspectives were explored +- `.factory/strategy/user-intent.md` — ground truth for user's ask + +Write the final plan to `.factory/strategy/current.md`. + +Required sections (in this order): +### Architecture + Merge from architecture strategy. Include component list and interfaces. +### Phased Plan + #### Phase 1: + - **What:** + - **Why:** + - **Acceptance criteria:** + - **Risks:** + #### Phase 2: + ... +### Acceptance Criteria + Full checklist from testing strategy. Each item must be: + - [ ] Specific enough for pass/fail verification + - Traceable to user intent (cite which part of user-intent.md) +### MVP Scope + From risk strategy. In vs deferred. +### Deferred Features + Items requiring human intervention or explicitly deferred. + +CRITICAL: The ### Acceptance Criteria section is the contract between +the builder and QA. It flows to adversarial testers who verify each +criterion independently. Make every item testable and unambiguous. +Read: .factory/strategy/user-intent.md +Write output to: .factory/strategy/current.md" --project "$PROJECT_PATH" --timeout 600 +``` + +```bash +# Artifact verification: synthesize_strategy +_vfail=0 +_f="$PROJECT_PATH/.factory/strategy/current.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: synthesize_strategy: .factory/strategy/current.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: synthesize_strategy: .factory/strategy/current.md is empty" && _vfail=1 +[ -f "$_f" ] && [ "$(wc -c < "$_f")" -lt 200 ] && echo "VERIFY FAIL: synthesize_strategy: .factory/strategy/current.md smaller than 200 bytes" && _vfail=1 +[ -f "$_f" ] && ! grep -qE '\#\#\#\ Phased\ Plan|\#\#\#\ Acceptance\ Criteria' "$_f" && echo "VERIFY FAIL: synthesize_strategy: .factory/strategy/current.md missing required sentinel (### Phased Plan, ### Acceptance Criteria)" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=synthesize_strategy" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: synthesize_strategy artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=synthesize_strategy" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" +``` +*(harness verification — DO NOT SKIP)* + +## Phase 7: Strategist — Design Doc + +```bash +factory agent strategist --task "You are a Technical Writer and Design Architect. Your job: take the structured +strategy at `.factory/strategy/current.md` and rewrite it as a proper +DESIGN DOCUMENT — a document a human reviewer can read end-to-end and +understand exactly what is being built, why, and how. + +Read: +- `.factory/strategy/current.md` — the structured strategy (your raw input) +- `.factory/strategy/user-intent.md` — the user's original idea and feedback + +Rewrite `.factory/strategy/current.md` IN PLACE. Replace the compressed +bullet points with a well-structured design document. + +Required sections (in this order): + +## What We're Building + Explain the project in 2-3 paragraphs of prose. What does it do? + Who is it for? What problem does it solve? Reference the user's + original words from user-intent.md. + +## Architecture + Describe the system architecture in full sentences. + Include a text-based architecture diagram using box-drawing characters: + ``` + ┌──────────┐ ┌──────────┐ ┌──────────┐ + │ Component│────▶│ Component│────▶│ Component│ + └──────────┘ └──────────┘ └──────────┘ + ``` + Explain each component — what it does, why it exists, how it connects. + +## How It Works + Walk through the user flow step by step. For a CLI, show example + invocations and expected output. For a web app, describe the user + journey screen by screen. For a library, show usage examples. + + This should read like a tutorial — someone unfamiliar with the project + should be able to follow along. + +## Phased Plan + For each phase, explain: + - What gets built in this phase and why this ordering + - What the user can do after this phase completes + - How to verify the phase worked (concrete test commands or checks) + + Use prose paragraphs, not just bullet points. Each phase should + read as a self-contained "chapter." + +## Acceptance Criteria + Present the full checklist, but group criteria by category and add + context for each. Explain WHY each criterion matters, not just what it is. + + Format: category heading, then checkbox items with brief explanation. + +## MVP Scope + What's in, what's deferred, and why. Explain the tradeoffs. + +## Deferred Features + Items deferred to future phases, with brief rationale. + +CRITICAL RULES: +- Write in full sentences and paragraphs, NOT bullet points +- The reader should understand the design WITHOUT reading any other file +- Include concrete examples (CLI invocations, API calls, code snippets) +- Architecture diagrams must use text/box-drawing characters +- Every technical choice must be explained — no unexplained jargon +- The document must be self-contained: a human reviewer reads ONLY this + file and decides whether to approve the design +Read: .factory/strategy/current.md, .factory/strategy/user-intent.md +Write output to: .factory/strategy/current.md" --project "$PROJECT_PATH" --timeout 600 +``` + +```bash +# Artifact verification: design_doc +_vfail=0 +_f="$PROJECT_PATH/.factory/strategy/current.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: design_doc: .factory/strategy/current.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: design_doc: .factory/strategy/current.md is empty" && _vfail=1 +[ -f "$_f" ] && [ "$(wc -c < "$_f")" -lt 500 ] && echo "VERIFY FAIL: design_doc: .factory/strategy/current.md smaller than 500 bytes" && _vfail=1 +[ -f "$_f" ] && ! grep -qE '\#\#\ What\ We're\ Building|\#\#\ Architecture|\#\#\ How\ It\ Works|\#\#\ Acceptance\ Criteria' "$_f" && echo "VERIFY FAIL: design_doc: .factory/strategy/current.md missing required sentinel (## What We're Building, ## Architecture, ## How It Works, ## Acceptance Criteria)" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=design_doc" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: design_doc artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=design_doc" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" +``` +*(harness verification — DO NOT SKIP)* + +### Steering Point — Strategy (User Approval) + +**This is a USER approval gate, NOT a CEO review gate. Do NOT self-approve.** + +Present the strategy/findings to the user by summarizing key points in your output. +Then explicitly ask the user: "Do you approve this plan, or do you have feedback?" + +**You MUST wait for the user's response before proceeding.** +- The user says "approve", "yes", "looks good", or similar → proceed to next step +- The user provides feedback or corrections → re-run the previous step incorporating their feedback +- Do NOT write a verdict file and auto-proceed — this gate requires human input + +*On RELOOP: return to `strategy_director` (max 3 iterations)* + +## Phase 8: Archivist Plan + +```bash +factory agent archivist --task "Archive the approved research and strategy. +Read: .factory/strategy/current.md +Write output to: .factory/archive/plan.md" --project "$PROJECT_PATH" --timeout 300 --model haiku & +``` +*(fire-and-forget — CEO continues immediately)* + +## Phase 9: Builder + +```bash +factory agent builder --task "Implement the next phase from .factory/strategy/current.md. Read the CEO's plan approval at .factory/reviews/ceo-verdict-strategist.md. Read CLAUDE.md and factory.md if they exist. Implement exactly what the current phase describes. Run tests. Commit changes and open a draft PR. +Read: .factory/strategy/current.md +Write output to: .factory/reviews/builder-latest.md" --project "$PROJECT_PATH" --timeout 1200 +``` + +```bash +# Artifact verification: builder +_vfail=0 +_f="$PROJECT_PATH/.factory/reviews/builder-latest.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: builder: .factory/reviews/builder-latest.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: builder: .factory/reviews/builder-latest.md is empty" && _vfail=1 +[ -f "$_f" ] && [ "$(wc -c < "$_f")" -lt 500 ] && echo "VERIFY FAIL: builder: .factory/reviews/builder-latest.md smaller than 500 bytes" && _vfail=1 +[ -f "$_f" ] && ! grep -qE 'commit' "$_f" && echo "VERIFY FAIL: builder: .factory/reviews/builder-latest.md missing required sentinel (commit)" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=builder" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: builder artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=builder" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" +``` +*(harness verification — DO NOT SKIP)* + +### CEO Review — Build + +Apply the CEO Review Gate protocol: +1. Read the agent output for the preceding step +2. Read artifacts: `.factory/reviews/builder-latest.md` +3. Assess: Read builder output. Check git log and diff. Does the work match the plan for this phase? If the Builder opened a PR, read it. REDIRECT if off-scope or missed key requirements. +4. Write verdict to `.factory/reviews/ceo-verdict-build.md` +5. **PROCEED** → continue to next step +6. **REDIRECT** → re-invoke the preceding agent with corrections (max 2) +7. **ABORT** → log failure and skip to archival + +*On RELOOP: return to `builder` (max 3 iterations)* + +## Phase 10: Qa (Parallel) + +Spawn 3 agents in parallel: + +```bash +factory agent health_checker --task "Execute health_checker task for the project. +Read: .factory/reviews/builder-latest.md, .factory/strategy/current.md +Write output to: .factory/reviews/health-check.md" --project "$PROJECT_PATH" --timeout 600 & +``` + +```bash +factory agent code_reviewer --task "Execute code_reviewer task for the project. +Read: .factory/reviews/builder-latest.md, .factory/strategy/current.md +Write output to: .factory/reviews/code-review.md" --project "$PROJECT_PATH" --timeout 900 & +``` + +```bash +factory agent ceo --task "You are the QA Director for this design session. + +Read: +- `.factory/strategy/current.md` — the design document with acceptance criteria +- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for +- `.factory/reviews/builder-latest.md` — what the builder implemented + +Your task has TWO phases: + +PHASE 1 — DESIGN QA APPROACHES +Analyze the acceptance criteria, the user's intent, and the builder's +implementation to identify K orthogonal testing approaches. + +Default approaches (adapt or replace based on the specific project): + - Happy path verification — does each acceptance criterion pass with + normal, expected inputs? + - Edge case & boundary testing — empty inputs, large inputs, special + characters, off-by-one, boundary conditions + - User intent verification — does the output match what the user + ACTUALLY asked for (from user-intent.md), not just the plan? + +You may add approaches specific to the implementation: + - Security testing (for auth, file I/O, user-facing APIs) + - Integration testing (for multi-component systems) + - Performance testing (for latency-sensitive features) + - Error handling testing (for systems with many failure modes) + - Concurrency testing (for parallel/async systems) + +K is NOT fixed — YOU decide based on the complexity of the acceptance +criteria and the nature of the implementation: + - Simple implementation (3-5 criteria): 2 testers + - Medium implementation (5-10 criteria): 3 testers + - Complex implementation (10+ criteria, security, integrations): 4-5 testers + +For each approach, design a TAILORED prompt — not a generic template. +Bad: "Test the implementation for edge cases" +Good: "Test the Obsidian wikilink resolver with these edge cases: + nested vault folders (vault/sub/note.md linking to ../other.md), + wikilinks with display text ([[note|display]]), + wikilinks to non-existent files, wikilinks with anchor + fragments ([[note#heading]]), and case-sensitivity mismatches." + +Write the QA plan to `.factory/reviews/qa-plan.json`: +```json +[ + {"approach": "...", "slug": "...", "prompt": "..."} +] +``` + +Constraints: +- Minimum 2 approaches, maximum 5 +- Each slug must be unique and kebab-case +- ALL acceptance criteria from current.md must be covered by at least + one tester's prompt +- One approach MUST verify user intent against user-intent.md +- Prompts must reference specific acceptance criteria and implementation details + +PHASE 2 — EXECUTE QA +For each approach in the plan, spawn an adversarial tester agent: +``` +factory agent adversarial_tester --task "" --project $PROJECT_PATH +``` + +Each tester writes to `.factory/reviews/adversarial--latest.md`. + +After ALL testers complete, review quality: +- Each adversarial report exists and has substantive findings +- All acceptance criteria are covered by at least one tester +- No tester missed its assigned focus area +- Critical findings are actually reproducible (spot-check) + +If a tester produced thin output, re-invoke it with a more specific prompt. + +Write a brief QA summary to the end of qa-plan.json noting which +approaches completed and any quality issues. +Read: .factory/reviews/builder-latest.md, .factory/strategy/current.md, .factory/strategy/user-intent.md +Write output to: .factory/reviews/qa-plan.json" --project "$PROJECT_PATH" --timeout 3600 & +``` + +```bash +wait +``` + +**Important:** Run ALL commands above in a **single** Bash tool call with timeout set to at least 3600 seconds. + +```bash +# Artifact verification: health_checker +_vfail=0 +_f="$PROJECT_PATH/.factory/reviews/health-check.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: health_checker: .factory/reviews/health-check.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: health_checker: .factory/reviews/health-check.md is empty" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=health_checker" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: health_checker artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=health_checker" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" + +# Artifact verification: code_reviewer +_vfail=0 +_f="$PROJECT_PATH/.factory/reviews/code-review.md" +[ ! -f "$_f" ] && echo "VERIFY FAIL: code_reviewer: .factory/reviews/code-review.md missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: code_reviewer: .factory/reviews/code-review.md is empty" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=code_reviewer" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: code_reviewer artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=code_reviewer" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" + +# Artifact verification: qa_director +_vfail=0 +_f="$PROJECT_PATH/.factory/reviews/qa-plan.json" +[ ! -f "$_f" ] && echo "VERIFY FAIL: qa_director: .factory/reviews/qa-plan.json missing" && _vfail=1 +[ -f "$_f" ] && [ ! -s "$_f" ] && echo "VERIFY FAIL: qa_director: .factory/reviews/qa-plan.json is empty" && _vfail=1 +[ -f "$_f" ] && [ "$(wc -c < "$_f")" -lt 20 ] && echo "VERIFY FAIL: qa_director: .factory/reviews/qa-plan.json smaller than 20 bytes" && _vfail=1 +[ "$_vfail" -ne 0 ] && echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_FAIL node=qa_director" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" && exit 1 +echo "VERIFY OK: qa_director artifacts validated" +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) VERIFY_OK node=qa_director" >> "$PROJECT_PATH/.factory/hooks/hook-log.txt" +``` +*(post-barrier harness verification — DO NOT SKIP)* + +## Barrier: Qa + +Wait for all parallel agents to complete: `health_checker`, `code_reviewer`, `qa_director` + +Read combined outputs: `.factory/reviews/code-review.md`, `.factory/reviews/health-check.md`, `.factory/reviews/qa-plan.json` + +## Step: Synthesize Qa + +Merges ALL adversarial-*-latest.md reports into one synthesized QA report. Uses glob pattern — works for any K testers. Findings caught by 2+ testers = HIGH confidence. Single-source findings surfaced but marked. Health checker and code reviewer reports included as pass-through. + +```bash +python3 -c "from pathlib import Path; import re, glob; project = '$PROJECT_PATH'; reports = []; for p in sorted(Path(f'{project}/.factory/reviews').glob('adversarial-*-latest.md')): slug = p.name.replace('-latest.md', '').replace('adversarial-', ''); reports.append((slug, p.read_text())); findings = {}; for tester_slug, text in reports: for line in text.splitlines(): stripped = line.strip(); if stripped.startswith('- ') or stripped.startswith('* '): key = re.sub(r'\s+', ' ', stripped[2:].strip().lower()[:80]); findings.setdefault(key, []).append(tester_slug); high = [(k, v) for k, v in findings.items() if len(v) >= 2]; medium = [(k, v) for k, v in findings.items() if len(v) == 1]; out = ['# Synthesized QA Report\n']; hc = Path(f'{project}/.factory/reviews/health-check.md'); cr = Path(f'{project}/.factory/reviews/code-review.md'); out.append('## Health Check\n'); out.append(hc.read_text() if hc.exists() else '(not available)'); out.append('\n## Code Review\n'); out.append(cr.read_text() if cr.exists() else '(not available)'); out.append('\n## High-Confidence Adversarial Findings (caught by 2+ testers)\n'); [out.append(f'- {k} (testers: {v})') for k, v in high]; if not high: out.append('- (none)'); out.append('\n## Medium-Confidence Adversarial Findings (single tester)\n'); [out.append(f'- {k} (tester: {v[0]})') for k, v in medium]; if not medium: out.append('- (none)'); out.append('\n## Raw Adversarial Reports\n'); [out.append(f'### Tester: {slug}\n{text}\n') for slug, text in reports]; Path(f'{project}/.factory/reviews/qa-synthesized.md').write_text('\n'.join(out)); print(f'Synthesized {len(high)} high + {len(medium)} medium findings from {len(reports)} adversarial reports')" +``` + +### CEO Review — Qa + +Apply the CEO Review Gate protocol: +1. Read the agent output for the preceding step +2. Read artifacts: `.factory/reviews/qa-synthesized.md`, `.factory/strategy/user-intent.md` +3. Assess: You are the CEO reviewing QA results. This is the final gate before merge. + +Read: +- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for +- `.factory/reviews/qa-synthesized.md` — merged QA report (health check, + code review, and synthesized adversarial findings) + +Decision framework: + +PROCEED if ALL of these hold: + 1. All acceptance criteria from current.md are verified PASS + 2. Health check passes (tests green, eval not regressed) + 3. No blocking code review issues + 4. No HIGH-confidence adversarial findings that violate user intent + +RELOOP to builder (max 3 iterations) if ANY of these hold: + 1. An acceptance criterion failed — cite which one + 2. Health check failed — cite which check + 3. Blocking review or HIGH adversarial findings + + When relooping, provide feedback mapped to SPECIFIC user requirements: + - "User asked for X (user-intent.md), but " + - "Acceptance criterion '' FAILED: " + + IMPORTANT: Append your reloop feedback to .factory/strategy/user-intent.md + under a new '## [timestamp] Reloop Feedback (Iteration N)' heading. + +HALT if: + - 3 reloops exhausted without resolution + - Fundamental design flaw that builder iterations cannot fix +4. Write verdict to `.factory/reviews/ceo-verdict-qa.md` +5. **PROCEED** → continue to next step +6. **REDIRECT** → re-invoke the preceding agent with corrections (max 2) +7. **ABORT** → log failure and skip to archival + +*On RELOOP: return to `builder` (max 3 iterations)* + +### CEO Review — Doc Freshness + +Apply the CEO Review Gate protocol: +1. Read the agent output for the preceding step +2. Read artifacts: `.factory/reviews/qa-synthesized.md` +3. Assess: Check the PR diff for documentation freshness. If public APIs, CLI commands, configuration options, or architecture were changed or added, corresponding documentation (README.md, CLAUDE.md, docstrings, --help text, or doc/ files) MUST be updated. PROCEED if docs are current or no doc-worthy changes exist. RELOOP to builder if documentation is stale — specify exactly which changes need doc updates. +4. Write verdict to `.factory/reviews/ceo-verdict-doc-freshness.md` +5. **PROCEED** → continue to next step +6. **REDIRECT** → re-invoke the preceding agent with corrections (max 2) +7. **ABORT** → log failure and skip to archival + +*On RELOOP: return to `builder` (max 3 iterations)* + +### Gate — Precheck (Automated) + +**MANDATORY:** Wait for the preceding agent to finish, then run this check BEFORE spawning the next agent. Do NOT run agents in parallel across this gate. + +```bash +factory precheck $PROJECT_PATH --score-before 0 --score-after 0 +``` + +- **PROCEED** (exit 0 / no FAIL in output) → continue to `archivist_build` +- **HALT** (exit non-zero / FAIL in output) → continue to `archivist_build` instead. + +## Phase 11: Archivist Build + +```bash +factory agent archivist --task "Archive the build phase results. +Read: .factory/reviews/qa-synthesized.md +Write output to: .factory/archive/build.md" --project "$PROJECT_PATH" --timeout 300 --model haiku & +``` +*(fire-and-forget — CEO continues immediately)* + +## Step: Spec Generate + +Generate the project specification via the gated spec-generate workflow. Runs non-blocking after archival. + +```bash +factory workflow run spec-generate $PROJECT_PATH +``` From 64a85ee2addf6687e1829947d69f1d938b5364a8 Mon Sep 17 00:00:00 2001 From: akashgit Date: Sat, 29 Aug 2026 15:09:29 -0400 Subject: [PATCH 2/9] refactor: extract design-v2 prompts to separate module, fix unused import - Extract all 7 prompt template constants from workflow.py into prompts.py - Remove unused `glob` import from synthesize_qa inline Python (Path.glob used instead) - Add comment on ADVERSARIAL_PROMPT noting it's used by QA Director when spawning testers Co-Authored-By: Claude Opus 4.6 (1M context) --- .../workflow/contributed/design_v2/prompts.py | 379 +++++++++++++++++ .../contributed/design_v2/workflow.py | 388 +----------------- 2 files changed, 389 insertions(+), 378 deletions(-) create mode 100644 factory/workflow/contributed/design_v2/prompts.py diff --git a/factory/workflow/contributed/design_v2/prompts.py b/factory/workflow/contributed/design_v2/prompts.py new file mode 100644 index 000000000..9751e695d --- /dev/null +++ b/factory/workflow/contributed/design_v2/prompts.py @@ -0,0 +1,379 @@ +"""Prompt templates for the design-v2 workflow.""" + +from __future__ import annotations + +RESEARCH_DIRECTOR_PROMPT = """\ +You are the Research Director for this design session. + +Read: +- `.factory/strategy/study-combined.md` — project study findings (observations + graph analysis) +- `.factory/strategy/user-intent.md` — the user's original idea + +Your task has TWO phases: + +PHASE 1 — DESIGN RESEARCH DIRECTIONS +Analyze the design space and identify N orthogonal research dimensions. +Default dimensions (adapt or replace based on the specific project): + - Similar projects / prior art + - Technology stack options + - Common pitfalls and failure modes +You may add dimensions specific to the domain (e.g., security, UX patterns, +data model constraints, compliance requirements, API design, concurrency). + +N is NOT fixed — YOU decide based on domain complexity: + - Simple CLI or library: 3 directions + - Web app with auth, database, UI: 4-5 directions + - Complex system with integrations, security, compliance: 5-7 directions + +For each direction, design a TAILORED prompt — not a generic template. +Bad: "Research similar projects and prior art" +Good: "Find existing link-checking tools that handle Obsidian-style wikilinks + ([[note]]) and image embeds (![[image.png]]). Compare how they resolve + relative paths vs vault-root-relative paths." + +Write the research plan to `.factory/strategy/research-plan.json`: +```json +[ + {"focus": "...", "slug": "...", "prompt": "..."} +] +``` + +Constraints: +- Minimum 3 directions, maximum 7 +- Each slug must be unique and kebab-case +- Prompts must be specific to THIS project, not generic templates + +PHASE 2 — EXECUTE RESEARCH +For each direction in the plan, spawn a researcher agent: +``` +factory agent researcher --task "" --project {project_path} +``` + +Each researcher writes to `.factory/strategy/research-.md`. + +After ALL researchers complete, review quality: +- Each research file exists and has substantive content (>50 bytes) +- No two reports cover the same ground excessively +- Key risks and opportunities are covered + +If a researcher produced thin output, re-invoke it with a more specific prompt. + +Write a brief research summary to the end of research-plan.json noting +which directions completed and any quality issues.""" + +STRATEGY_DIRECTOR_PROMPT = """\ +You are the Strategy Director for this design session. + +Read: +- ALL research reports at `.factory/strategy/research-*.md` +- `.factory/strategy/research-plan.json` — which research directions were explored +- `.factory/strategy/user-intent.md` — the user's original idea +- `.factory/strategy/study-combined.md` — project context + +Your task has TWO phases: + +PHASE 1 — DESIGN STRATEGY PERSPECTIVES +Analyze the research findings and user intent to identify M strategy +perspectives this project needs. + +Default perspectives (adapt or replace based on the specific project): + - Architecture strategy — how to build it (components, phases, tech choices) + - Testing/verification strategy — how to verify it works (acceptance criteria, + test plan, edge cases) + - Risk/scope strategy — what to cut, what's hard, what breaks + +You may add perspectives specific to the domain: + - Security strategy (for auth-heavy projects) + - Data modeling strategy (for data-heavy projects) + - API design strategy (for API-first projects) + - Performance strategy (for latency-sensitive systems) + +M is NOT fixed — YOU decide based on project complexity: + - Simple project: 2-3 perspectives + - Medium project: 3-4 perspectives + - Complex project: 4-5 perspectives + +For each perspective, design a TAILORED prompt. +Bad: "Create an architecture strategy" +Good: "Design the architecture for a markdown link checker CLI. The core + challenge is resolving Obsidian wikilinks against a configurable vault + root while also supporting standard URLs with redirect following. + Research shows handles HTTP well but nothing handles + wikilinks — design that component from scratch." + +Write the strategy plan to `.factory/strategy/strategy-plan.json`: +```json +[ + {"perspective": "...", "slug": "...", "prompt": "..."} +] +``` + +Constraints: +- Minimum 2 perspectives, maximum 5 +- Each slug must be unique and kebab-case +- One perspective MUST cover testing/verification with explicit acceptance criteria +- Prompts must reference specific findings from the research reports + +PHASE 2 — EXECUTE STRATEGIES +For each perspective in the plan, spawn a strategist agent: +``` +factory agent strategist --task "" --project {project_path} +``` + +Each strategist writes to `.factory/strategy/strategy-.md`. + +After ALL strategists complete, review quality: +- Each strategy file exists and has substantive content (>100 bytes) +- The testing strategy has a `### Acceptance Criteria` section with checkboxes +- Architecture strategy cites research findings +- No critical perspective is missing + +If a strategist produced thin output, re-invoke it with a more specific prompt. + +Write a brief strategy summary to the end of strategy-plan.json noting +which perspectives completed and any quality issues.""" + +SYNTHESIZE_STRATEGY_PROMPT = """\ +You are the Strategy Synthesizer. Compile one final plan from all +strategy inputs. + +Read: +- ALL strategy files at `.factory/strategy/strategy-*.md` +- `.factory/strategy/strategy-plan.json` — which perspectives were explored +- `.factory/strategy/user-intent.md` — ground truth for user's ask + +Write the final plan to `.factory/strategy/current.md`. + +Required sections (in this order): +### Architecture + Merge from architecture strategy. Include component list and interfaces. +### Phased Plan + #### Phase 1: + - **What:** + - **Why:** + - **Acceptance criteria:** + - **Risks:** + #### Phase 2: + ... +### Acceptance Criteria + Full checklist from testing strategy. Each item must be: + - [ ] Specific enough for pass/fail verification + - Traceable to user intent (cite which part of user-intent.md) +### MVP Scope + From risk strategy. In vs deferred. +### Deferred Features + Items requiring human intervention or explicitly deferred. + +CRITICAL: The ### Acceptance Criteria section is the contract between +the builder and QA. It flows to adversarial testers who verify each +criterion independently. Make every item testable and unambiguous.""" + +DESIGN_DOC_PROMPT = """\ +You are a Technical Writer and Design Architect. Your job: take the structured +strategy at `.factory/strategy/current.md` and rewrite it as a proper +DESIGN DOCUMENT — a document a human reviewer can read end-to-end and +understand exactly what is being built, why, and how. + +Read: +- `.factory/strategy/current.md` — the structured strategy (your raw input) +- `.factory/strategy/user-intent.md` — the user's original idea and feedback + +Rewrite `.factory/strategy/current.md` IN PLACE. Replace the compressed +bullet points with a well-structured design document. + +Required sections (in this order): + +## What We're Building + Explain the project in 2-3 paragraphs of prose. What does it do? + Who is it for? What problem does it solve? Reference the user's + original words from user-intent.md. + +## Architecture + Describe the system architecture in full sentences. + Include a text-based architecture diagram using box-drawing characters: + ``` + ┌──────────┐ ┌──────────┐ ┌──────────┐ + │ Component│────▶│ Component│────▶│ Component│ + └──────────┘ └──────────┘ └──────────┘ + ``` + Explain each component — what it does, why it exists, how it connects. + +## How It Works + Walk through the user flow step by step. For a CLI, show example + invocations and expected output. For a web app, describe the user + journey screen by screen. For a library, show usage examples. + + This should read like a tutorial — someone unfamiliar with the project + should be able to follow along. + +## Phased Plan + For each phase, explain: + - What gets built in this phase and why this ordering + - What the user can do after this phase completes + - How to verify the phase worked (concrete test commands or checks) + + Use prose paragraphs, not just bullet points. Each phase should + read as a self-contained "chapter." + +## Acceptance Criteria + Present the full checklist, but group criteria by category and add + context for each. Explain WHY each criterion matters, not just what it is. + + Format: category heading, then checkbox items with brief explanation. + +## MVP Scope + What's in, what's deferred, and why. Explain the tradeoffs. + +## Deferred Features + Items deferred to future phases, with brief rationale. + +CRITICAL RULES: +- Write in full sentences and paragraphs, NOT bullet points +- The reader should understand the design WITHOUT reading any other file +- Include concrete examples (CLI invocations, API calls, code snippets) +- Architecture diagrams must use text/box-drawing characters +- Every technical choice must be explained — no unexplained jargon +- The document must be self-contained: a human reviewer reads ONLY this + file and decides whether to approve the design""" + +QA_DIRECTOR_PROMPT = """\ +You are the QA Director for this design session. + +Read: +- `.factory/strategy/current.md` — the design document with acceptance criteria +- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for +- `.factory/reviews/builder-latest.md` — what the builder implemented + +Your task has TWO phases: + +PHASE 1 — DESIGN QA APPROACHES +Analyze the acceptance criteria, the user's intent, and the builder's +implementation to identify K orthogonal testing approaches. + +Default approaches (adapt or replace based on the specific project): + - Happy path verification — does each acceptance criterion pass with + normal, expected inputs? + - Edge case & boundary testing — empty inputs, large inputs, special + characters, off-by-one, boundary conditions + - User intent verification — does the output match what the user + ACTUALLY asked for (from user-intent.md), not just the plan? + +You may add approaches specific to the implementation: + - Security testing (for auth, file I/O, user-facing APIs) + - Integration testing (for multi-component systems) + - Performance testing (for latency-sensitive features) + - Error handling testing (for systems with many failure modes) + - Concurrency testing (for parallel/async systems) + +K is NOT fixed — YOU decide based on the complexity of the acceptance +criteria and the nature of the implementation: + - Simple implementation (3-5 criteria): 2 testers + - Medium implementation (5-10 criteria): 3 testers + - Complex implementation (10+ criteria, security, integrations): 4-5 testers + +For each approach, design a TAILORED prompt — not a generic template. +Bad: "Test the implementation for edge cases" +Good: "Test the Obsidian wikilink resolver with these edge cases: + nested vault folders (vault/sub/note.md linking to ../other.md), + wikilinks with display text ([[note|display]]), + wikilinks to non-existent files, wikilinks with anchor + fragments ([[note#heading]]), and case-sensitivity mismatches." + +Write the QA plan to `.factory/reviews/qa-plan.json`: +```json +[ + {"approach": "...", "slug": "...", "prompt": "..."} +] +``` + +Constraints: +- Minimum 2 approaches, maximum 5 +- Each slug must be unique and kebab-case +- ALL acceptance criteria from current.md must be covered by at least + one tester's prompt +- One approach MUST verify user intent against user-intent.md +- Prompts must reference specific acceptance criteria and implementation details + +PHASE 2 — EXECUTE QA +For each approach in the plan, spawn an adversarial tester agent: +``` +factory agent adversarial_tester --task "" --project {project_path} +``` + +Each tester writes to `.factory/reviews/adversarial--latest.md`. + +After ALL testers complete, review quality: +- Each adversarial report exists and has substantive findings +- All acceptance criteria are covered by at least one tester +- No tester missed its assigned focus area +- Critical findings are actually reproducible (spot-check) + +If a tester produced thin output, re-invoke it with a more specific prompt. + +Write a brief QA summary to the end of qa-plan.json noting which +approaches completed and any quality issues.""" + +# Used by the QA Director when spawning adversarial testers (referenced in QA_DIRECTOR_PROMPT). +ADVERSARIAL_PROMPT = """\ +You are an adversarial tester. Your job: break the implementation. + +Read: +- `.factory/strategy/current.md` — the design document and acceptance criteria +- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for +- `.factory/reviews/builder-latest.md` — builder's work summary +- Source code changes (use git diff) + +Procedure: +1. Read the acceptance criteria from current.md +2. For each criterion, attempt to verify it by running the code +3. Try edge cases, invalid inputs, boundary conditions +4. Check that the implementation matches user intent, not just the plan +5. Look for security issues, error handling gaps, missing validations + +Output format: +# Adversarial Test Report + +## Acceptance Criteria Verification +- [ ] Criterion 1: PASS/FAIL — evidence +- [ ] Criterion 2: PASS/FAIL — evidence + +## Edge Case Findings +- Finding: + - Steps to reproduce: + - Expected: + - Actual: + +## User Intent Verification +- Does the output match what the user asked for? Evidence: <...>""" + +GATE_QA_PROMPT = """\ +You are the CEO reviewing QA results. This is the final gate before merge. + +Read: +- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for +- `.factory/reviews/qa-synthesized.md` — merged QA report (health check, + code review, and synthesized adversarial findings) + +Decision framework: + +PROCEED if ALL of these hold: + 1. All acceptance criteria from current.md are verified PASS + 2. Health check passes (tests green, eval not regressed) + 3. No blocking code review issues + 4. No HIGH-confidence adversarial findings that violate user intent + +RELOOP to builder (max 3 iterations) if ANY of these hold: + 1. An acceptance criterion failed — cite which one + 2. Health check failed — cite which check + 3. Blocking review or HIGH adversarial findings + + When relooping, provide feedback mapped to SPECIFIC user requirements: + - "User asked for X (user-intent.md), but " + - "Acceptance criterion '' FAILED: " + + IMPORTANT: Append your reloop feedback to .factory/strategy/user-intent.md + under a new '## [timestamp] Reloop Feedback (Iteration N)' heading. + +HALT if: + - 3 reloops exhausted without resolution + - Fundamental design flaw that builder iterations cannot fix""" diff --git a/factory/workflow/contributed/design_v2/workflow.py b/factory/workflow/contributed/design_v2/workflow.py index ad1e5a628..e4e918bfc 100644 --- a/factory/workflow/contributed/design_v2/workflow.py +++ b/factory/workflow/contributed/design_v2/workflow.py @@ -20,6 +20,15 @@ Workflow, ) +from .prompts import ( + DESIGN_DOC_PROMPT, + GATE_QA_PROMPT, + QA_DIRECTOR_PROMPT, + RESEARCH_DIRECTOR_PROMPT, + STRATEGY_DIRECTOR_PROMPT, + SYNTHESIZE_STRATEGY_PROMPT, +) + meta = { "name": "design-v2", "description": ( @@ -28,383 +37,6 @@ ), } -# ── Prompt templates ──────────────────────────────────────────── - -RESEARCH_DIRECTOR_PROMPT = """\ -You are the Research Director for this design session. - -Read: -- `.factory/strategy/study-combined.md` — project study findings (observations + graph analysis) -- `.factory/strategy/user-intent.md` — the user's original idea - -Your task has TWO phases: - -PHASE 1 — DESIGN RESEARCH DIRECTIONS -Analyze the design space and identify N orthogonal research dimensions. -Default dimensions (adapt or replace based on the specific project): - - Similar projects / prior art - - Technology stack options - - Common pitfalls and failure modes -You may add dimensions specific to the domain (e.g., security, UX patterns, -data model constraints, compliance requirements, API design, concurrency). - -N is NOT fixed — YOU decide based on domain complexity: - - Simple CLI or library: 3 directions - - Web app with auth, database, UI: 4-5 directions - - Complex system with integrations, security, compliance: 5-7 directions - -For each direction, design a TAILORED prompt — not a generic template. -Bad: "Research similar projects and prior art" -Good: "Find existing link-checking tools that handle Obsidian-style wikilinks - ([[note]]) and image embeds (![[image.png]]). Compare how they resolve - relative paths vs vault-root-relative paths." - -Write the research plan to `.factory/strategy/research-plan.json`: -```json -[ - {"focus": "...", "slug": "...", "prompt": "..."} -] -``` - -Constraints: -- Minimum 3 directions, maximum 7 -- Each slug must be unique and kebab-case -- Prompts must be specific to THIS project, not generic templates - -PHASE 2 — EXECUTE RESEARCH -For each direction in the plan, spawn a researcher agent: -``` -factory agent researcher --task "" --project {project_path} -``` - -Each researcher writes to `.factory/strategy/research-.md`. - -After ALL researchers complete, review quality: -- Each research file exists and has substantive content (>50 bytes) -- No two reports cover the same ground excessively -- Key risks and opportunities are covered - -If a researcher produced thin output, re-invoke it with a more specific prompt. - -Write a brief research summary to the end of research-plan.json noting -which directions completed and any quality issues.""" - -STRATEGY_DIRECTOR_PROMPT = """\ -You are the Strategy Director for this design session. - -Read: -- ALL research reports at `.factory/strategy/research-*.md` -- `.factory/strategy/research-plan.json` — which research directions were explored -- `.factory/strategy/user-intent.md` — the user's original idea -- `.factory/strategy/study-combined.md` — project context - -Your task has TWO phases: - -PHASE 1 — DESIGN STRATEGY PERSPECTIVES -Analyze the research findings and user intent to identify M strategy -perspectives this project needs. - -Default perspectives (adapt or replace based on the specific project): - - Architecture strategy — how to build it (components, phases, tech choices) - - Testing/verification strategy — how to verify it works (acceptance criteria, - test plan, edge cases) - - Risk/scope strategy — what to cut, what's hard, what breaks - -You may add perspectives specific to the domain: - - Security strategy (for auth-heavy projects) - - Data modeling strategy (for data-heavy projects) - - API design strategy (for API-first projects) - - Performance strategy (for latency-sensitive systems) - -M is NOT fixed — YOU decide based on project complexity: - - Simple project: 2-3 perspectives - - Medium project: 3-4 perspectives - - Complex project: 4-5 perspectives - -For each perspective, design a TAILORED prompt. -Bad: "Create an architecture strategy" -Good: "Design the architecture for a markdown link checker CLI. The core - challenge is resolving Obsidian wikilinks against a configurable vault - root while also supporting standard URLs with redirect following. - Research shows handles HTTP well but nothing handles - wikilinks — design that component from scratch." - -Write the strategy plan to `.factory/strategy/strategy-plan.json`: -```json -[ - {"perspective": "...", "slug": "...", "prompt": "..."} -] -``` - -Constraints: -- Minimum 2 perspectives, maximum 5 -- Each slug must be unique and kebab-case -- One perspective MUST cover testing/verification with explicit acceptance criteria -- Prompts must reference specific findings from the research reports - -PHASE 2 — EXECUTE STRATEGIES -For each perspective in the plan, spawn a strategist agent: -``` -factory agent strategist --task "" --project {project_path} -``` - -Each strategist writes to `.factory/strategy/strategy-.md`. - -After ALL strategists complete, review quality: -- Each strategy file exists and has substantive content (>100 bytes) -- The testing strategy has a `### Acceptance Criteria` section with checkboxes -- Architecture strategy cites research findings -- No critical perspective is missing - -If a strategist produced thin output, re-invoke it with a more specific prompt. - -Write a brief strategy summary to the end of strategy-plan.json noting -which perspectives completed and any quality issues.""" - -SYNTHESIZE_STRATEGY_PROMPT = """\ -You are the Strategy Synthesizer. Compile one final plan from all -strategy inputs. - -Read: -- ALL strategy files at `.factory/strategy/strategy-*.md` -- `.factory/strategy/strategy-plan.json` — which perspectives were explored -- `.factory/strategy/user-intent.md` — ground truth for user's ask - -Write the final plan to `.factory/strategy/current.md`. - -Required sections (in this order): -### Architecture - Merge from architecture strategy. Include component list and interfaces. -### Phased Plan - #### Phase 1: - - **What:** - - **Why:** - - **Acceptance criteria:** - - **Risks:** - #### Phase 2: - ... -### Acceptance Criteria - Full checklist from testing strategy. Each item must be: - - [ ] Specific enough for pass/fail verification - - Traceable to user intent (cite which part of user-intent.md) -### MVP Scope - From risk strategy. In vs deferred. -### Deferred Features - Items requiring human intervention or explicitly deferred. - -CRITICAL: The ### Acceptance Criteria section is the contract between -the builder and QA. It flows to adversarial testers who verify each -criterion independently. Make every item testable and unambiguous.""" - -DESIGN_DOC_PROMPT = """\ -You are a Technical Writer and Design Architect. Your job: take the structured -strategy at `.factory/strategy/current.md` and rewrite it as a proper -DESIGN DOCUMENT — a document a human reviewer can read end-to-end and -understand exactly what is being built, why, and how. - -Read: -- `.factory/strategy/current.md` — the structured strategy (your raw input) -- `.factory/strategy/user-intent.md` — the user's original idea and feedback - -Rewrite `.factory/strategy/current.md` IN PLACE. Replace the compressed -bullet points with a well-structured design document. - -Required sections (in this order): - -## What We're Building - Explain the project in 2-3 paragraphs of prose. What does it do? - Who is it for? What problem does it solve? Reference the user's - original words from user-intent.md. - -## Architecture - Describe the system architecture in full sentences. - Include a text-based architecture diagram using box-drawing characters: - ``` - ┌──────────┐ ┌──────────┐ ┌──────────┐ - │ Component│────▶│ Component│────▶│ Component│ - └──────────┘ └──────────┘ └──────────┘ - ``` - Explain each component — what it does, why it exists, how it connects. - -## How It Works - Walk through the user flow step by step. For a CLI, show example - invocations and expected output. For a web app, describe the user - journey screen by screen. For a library, show usage examples. - - This should read like a tutorial — someone unfamiliar with the project - should be able to follow along. - -## Phased Plan - For each phase, explain: - - What gets built in this phase and why this ordering - - What the user can do after this phase completes - - How to verify the phase worked (concrete test commands or checks) - - Use prose paragraphs, not just bullet points. Each phase should - read as a self-contained "chapter." - -## Acceptance Criteria - Present the full checklist, but group criteria by category and add - context for each. Explain WHY each criterion matters, not just what it is. - - Format: category heading, then checkbox items with brief explanation. - -## MVP Scope - What's in, what's deferred, and why. Explain the tradeoffs. - -## Deferred Features - Items deferred to future phases, with brief rationale. - -CRITICAL RULES: -- Write in full sentences and paragraphs, NOT bullet points -- The reader should understand the design WITHOUT reading any other file -- Include concrete examples (CLI invocations, API calls, code snippets) -- Architecture diagrams must use text/box-drawing characters -- Every technical choice must be explained — no unexplained jargon -- The document must be self-contained: a human reviewer reads ONLY this - file and decides whether to approve the design""" - -QA_DIRECTOR_PROMPT = """\ -You are the QA Director for this design session. - -Read: -- `.factory/strategy/current.md` — the design document with acceptance criteria -- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for -- `.factory/reviews/builder-latest.md` — what the builder implemented - -Your task has TWO phases: - -PHASE 1 — DESIGN QA APPROACHES -Analyze the acceptance criteria, the user's intent, and the builder's -implementation to identify K orthogonal testing approaches. - -Default approaches (adapt or replace based on the specific project): - - Happy path verification — does each acceptance criterion pass with - normal, expected inputs? - - Edge case & boundary testing — empty inputs, large inputs, special - characters, off-by-one, boundary conditions - - User intent verification — does the output match what the user - ACTUALLY asked for (from user-intent.md), not just the plan? - -You may add approaches specific to the implementation: - - Security testing (for auth, file I/O, user-facing APIs) - - Integration testing (for multi-component systems) - - Performance testing (for latency-sensitive features) - - Error handling testing (for systems with many failure modes) - - Concurrency testing (for parallel/async systems) - -K is NOT fixed — YOU decide based on the complexity of the acceptance -criteria and the nature of the implementation: - - Simple implementation (3-5 criteria): 2 testers - - Medium implementation (5-10 criteria): 3 testers - - Complex implementation (10+ criteria, security, integrations): 4-5 testers - -For each approach, design a TAILORED prompt — not a generic template. -Bad: "Test the implementation for edge cases" -Good: "Test the Obsidian wikilink resolver with these edge cases: - nested vault folders (vault/sub/note.md linking to ../other.md), - wikilinks with display text ([[note|display]]), - wikilinks to non-existent files, wikilinks with anchor - fragments ([[note#heading]]), and case-sensitivity mismatches." - -Write the QA plan to `.factory/reviews/qa-plan.json`: -```json -[ - {"approach": "...", "slug": "...", "prompt": "..."} -] -``` - -Constraints: -- Minimum 2 approaches, maximum 5 -- Each slug must be unique and kebab-case -- ALL acceptance criteria from current.md must be covered by at least - one tester's prompt -- One approach MUST verify user intent against user-intent.md -- Prompts must reference specific acceptance criteria and implementation details - -PHASE 2 — EXECUTE QA -For each approach in the plan, spawn an adversarial tester agent: -``` -factory agent adversarial_tester --task "" --project {project_path} -``` - -Each tester writes to `.factory/reviews/adversarial--latest.md`. - -After ALL testers complete, review quality: -- Each adversarial report exists and has substantive findings -- All acceptance criteria are covered by at least one tester -- No tester missed its assigned focus area -- Critical findings are actually reproducible (spot-check) - -If a tester produced thin output, re-invoke it with a more specific prompt. - -Write a brief QA summary to the end of qa-plan.json noting which -approaches completed and any quality issues.""" - -ADVERSARIAL_PROMPT = """\ -You are an adversarial tester. Your job: break the implementation. - -Read: -- `.factory/strategy/current.md` — the design document and acceptance criteria -- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for -- `.factory/reviews/builder-latest.md` — builder's work summary -- Source code changes (use git diff) - -Procedure: -1. Read the acceptance criteria from current.md -2. For each criterion, attempt to verify it by running the code -3. Try edge cases, invalid inputs, boundary conditions -4. Check that the implementation matches user intent, not just the plan -5. Look for security issues, error handling gaps, missing validations - -Output format: -# Adversarial Test Report - -## Acceptance Criteria Verification -- [ ] Criterion 1: PASS/FAIL — evidence -- [ ] Criterion 2: PASS/FAIL — evidence - -## Edge Case Findings -- Finding: - - Steps to reproduce: - - Expected: - - Actual: - -## User Intent Verification -- Does the output match what the user asked for? Evidence: <...>""" - -GATE_QA_PROMPT = """\ -You are the CEO reviewing QA results. This is the final gate before merge. - -Read: -- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for -- `.factory/reviews/qa-synthesized.md` — merged QA report (health check, - code review, and synthesized adversarial findings) - -Decision framework: - -PROCEED if ALL of these hold: - 1. All acceptance criteria from current.md are verified PASS - 2. Health check passes (tests green, eval not regressed) - 3. No blocking code review issues - 4. No HIGH-confidence adversarial findings that violate user intent - -RELOOP to builder (max 3 iterations) if ANY of these hold: - 1. An acceptance criterion failed — cite which one - 2. Health check failed — cite which check - 3. Blocking review or HIGH adversarial findings - - When relooping, provide feedback mapped to SPECIFIC user requirements: - - "User asked for X (user-intent.md), but " - - "Acceptance criterion '' FAILED: " - - IMPORTANT: Append your reloop feedback to .factory/strategy/user-intent.md - under a new '## [timestamp] Reloop Feedback (Iteration N)' heading. - -HALT if: - - 3 reloops exhausted without resolution - - Fundamental design flaw that builder iterations cannot fix""" - def workflow() -> Workflow: """Build the design-v2 workflow graph.""" @@ -614,7 +246,7 @@ def workflow() -> Workflow: command=( 'python3 -c "' "from pathlib import Path; " - "import re, glob; " + "import re; " "project = '{project_path}'; " "reports = []; " "for p in sorted(Path(f'{project}/.factory/reviews').glob(" From 04b7f29cc3067d070da6da7c6e99eab70a704d1c Mon Sep 17 00:00:00 2001 From: akashgit Date: Sat, 29 Aug 2026 15:23:03 -0400 Subject: [PATCH 3/9] fix: recognize design-v2 (and contributed workflows) in CLI mode routing The CLI mode validation only checked CEO_MODES and project-local workflows, ignoring builtin registry entries like design-v2. Two fixes: 1. Add "design-v2" to CEO_MODES in _helpers.py 2. Change fallback validation from project-only to all workflow registry entries, so any registered workflow (builtin, contributed, project) is automatically valid 3. Add "design-v2" alongside "design" in all mode routing checks so it supports --focus, --auto-approve, --from-plan, --just-plan Closes #1392 Co-Authored-By: Claude Opus 4.6 (1M context) --- factory/cli/_ceo_helpers.py | 29 +++++++++++++++-------------- factory/cli/_helpers.py | 1 + 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/factory/cli/_ceo_helpers.py b/factory/cli/_ceo_helpers.py index 3bf886efd..f3ccb019c 100644 --- a/factory/cli/_ceo_helpers.py +++ b/factory/cli/_ceo_helpers.py @@ -170,8 +170,8 @@ def _validate_ceo_flags( raw_path = getattr(args, "path", None) project_path = Path(raw_path).resolve() if raw_path else Path.cwd() entries = WorkflowRegistry.discover(project_path) - project_entries = {n for n, e in entries.items() if e.source == "project"} - if mode not in project_entries: + all_workflow_entries = set(entries.keys()) + if mode not in all_workflow_entries: print( f"Error: unknown mode '{mode}'. " f"Not a built-in mode and not found in project workflows at " @@ -193,12 +193,12 @@ def _validate_ceo_flags( from_plan: str | None = getattr(args, "from_plan", None) just_plan: bool = getattr(args, "just_plan", False) - if auto_approve and mode != "design": + if auto_approve and mode not in ("design", "design-v2"): print("Error: --auto-approve only applies to --mode design", file=sys.stderr) return 1 if just_plan: - if mode != "design": + if mode not in ("design", "design-v2"): print("Error: --just-plan requires --mode design", file=sys.stderr) return 1 if from_plan: @@ -209,7 +209,7 @@ def _validate_ceo_flags( return 1 if from_plan: - if mode != "design": + if mode not in ("design", "design-v2"): print("Error: --from-plan requires --mode design", file=sys.stderr) return 1 if focus: @@ -255,10 +255,10 @@ def _validate_ceo_flags( return 1 _design_is_existing = ( - mode == "design" and raw_path and _safe_is_dir(Path(raw_path).expanduser().resolve()) + mode in ("design", "design-v2") and raw_path and _safe_is_dir(Path(raw_path).expanduser().resolve()) ) - if mode == "design": + if mode in ("design", "design-v2"): if auto_approve: headless = True elif headless: @@ -354,7 +354,7 @@ def _resolve_ceo_project( context: str | None = None _design_is_existing = ( - mode == "design" and raw_path and _safe_is_dir(Path(raw_path).expanduser().resolve()) + mode in ("design", "design-v2") and raw_path and _safe_is_dir(Path(raw_path).expanduser().resolve()) ) if mode == "create": @@ -377,10 +377,10 @@ def _resolve_ceo_project( if m.group(1) in registered: update_existing_mode = m.group(1) create_description = m.group(2).strip() - elif mode == "design" and _design_is_existing: + elif mode in ("design", "design-v2") and _design_is_existing: project_path, context = _resolve_input(raw_path, dir_name=dir_name) design_existing = True - elif mode == "design": + elif mode in ("design", "design-v2"): resolved_file = Path(raw_path).expanduser() if _safe_is_file(resolved_file): design_idea = resolved_file.read_text() @@ -485,6 +485,7 @@ def _validate_late_flags( and mode not in ( "design", + "design-v2", "research", "create", "evolve", @@ -639,8 +640,8 @@ def _execute_ceo( ) if mode == "create": ceo_mode = "create" - elif mode == "design": - ceo_mode = "design" + elif mode in ("design", "design-v2"): + ceo_mode = mode elif interactive: ceo_mode = "design" else: @@ -915,7 +916,7 @@ def _run_headless( min_growth=min_growth, max_new=max_new, branch=branch, - already_improved=mode in ("design", "meta") or discover_only, + already_improved=mode in ("design", "design-v2", "meta") or discover_only, model=model, no_github=no_github, use_profile=use_profile, @@ -968,7 +969,7 @@ def _run_headless( min_growth=min_growth, max_new=max_new, branch=branch, - already_improved=mode in ("design", "meta") or discover_only, + already_improved=mode in ("design", "design-v2", "meta") or discover_only, model=model, no_github=no_github, use_profile=use_profile, diff --git a/factory/cli/_helpers.py b/factory/cli/_helpers.py index 305e32c6a..02f539624 100644 --- a/factory/cli/_helpers.py +++ b/factory/cli/_helpers.py @@ -43,6 +43,7 @@ "evolve", "deep-research", "outer-loop", + "design-v2", ] From 0aa3625681c739e27a67655d0118a8eb4bd8be1a Mon Sep 17 00:00:00 2001 From: akashgit Date: Sat, 29 Aug 2026 15:24:57 -0400 Subject: [PATCH 4/9] =?UTF-8?q?fix:=20remove=20design-v2=20from=20CEO=5FMO?= =?UTF-8?q?DES=20=E2=80=94=20discovered=20via=20WorkflowRegistry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit design-v2 is a builtin workflow discovered by WorkflowRegistry.discover() and passes the fallback check in _ceo_helpers.py without needing an explicit CEO_MODES entry. The hardcoded entry was redundant. Co-Authored-By: Claude Opus 4.6 (1M context) --- factory/cli/_helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/factory/cli/_helpers.py b/factory/cli/_helpers.py index 02f539624..305e32c6a 100644 --- a/factory/cli/_helpers.py +++ b/factory/cli/_helpers.py @@ -43,7 +43,6 @@ "evolve", "deep-research", "outer-loop", - "design-v2", ] From 645f9482241e7da9c9be58ae57404c1464a333e2 Mon Sep 17 00:00:00 2001 From: akashgit Date: Sat, 29 Aug 2026 16:01:55 -0400 Subject: [PATCH 5/9] fix: wire auto_approve through to CEO task string for design/design-v2 modes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When --auto-approve is passed, the CEO now receives instructions to act as the user at approval gates — reviewing plans against user-intent.md and making PROCEED/feedback decisions instead of waiting for human input. Co-Authored-By: Claude Opus 4.6 (1M context) --- factory/cli/_ceo_helpers.py | 1 + factory/cli/_task_builder.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/factory/cli/_ceo_helpers.py b/factory/cli/_ceo_helpers.py index f3ccb019c..638e8ff4e 100644 --- a/factory/cli/_ceo_helpers.py +++ b/factory/cli/_ceo_helpers.py @@ -712,6 +712,7 @@ def _execute_ceo( from_plan=resolved_plan.plan if resolved_plan else None, from_plan_feedback=resolved_plan.feedback if resolved_plan else None, just_plan=just_plan, + auto_approve=auto_approve, ) session_name = _derive_session_name( diff --git a/factory/cli/_task_builder.py b/factory/cli/_task_builder.py index f57b7fdb8..2790935e7 100644 --- a/factory/cli/_task_builder.py +++ b/factory/cli/_task_builder.py @@ -191,6 +191,7 @@ def _build_ceo_task( from_plan: str | None = None, from_plan_feedback: list[str] | None = None, just_plan: bool = False, + auto_approve: bool = False, ) -> str: """Build the CEO agent task string from mode and optional context.""" shown_mode = display_mode if display_mode is not None else mode @@ -491,6 +492,20 @@ def _build_ceo_task( f"Do NOT skip the review pipeline. Do NOT abbreviate any step.\n" ) + if auto_approve: + task += ( + "\n\n## Auto-Approve Mode\n\n" + "auto_approve: true\n\n" + "At user approval gates (like gate_strategy), you act as the user:\n" + "1. Read the plan at .factory/strategy/current.md\n" + "2. Read the user's original intent at .factory/strategy/user-intent.md\n" + "3. Compare: does the plan match what the user asked for?\n" + "4. If YES: approve and proceed (say \"Approved\" and continue to the next step)\n" + "5. If NO: provide specific feedback about what's missing or wrong, then reloop\n\n" + "You are the CEO acting on behalf of the user. Apply judgment — approve good plans, " + "reject bad ones. Do NOT blindly approve everything. Do NOT wait for human input.\n" + ) + if clean_pr: task += ( "\n\n## Clean PR Mode\n\n" From 2f8feae18edc39095538167fa6b8d5a4f91bba3d Mon Sep 17 00:00:00 2001 From: akashgit Date: Sat, 29 Aug 2026 16:58:53 -0400 Subject: [PATCH 6/9] fix: address 5 code review issues on design-v2 workflow (#1392) - Add design-v2 to mode checks in run.py (auto_approve, focus, skip_improve) - Fix init_user_intent FOCUS env var: read from FACTORY_IDEA env, backlog.md fallback, skip if user-intent.md already exists - Fix single-quote injection in init_user_intent and synthesize_qa by passing project_path via sys.argv instead of string interpolation - Update error messages in _ceo_helpers.py to mention design-v2 - Remove dead ADVERSARIAL_PROMPT constant, fold output format into QA_DIRECTOR_PROMPT Co-Authored-By: Claude Opus 4.6 (1M context) --- factory/cli/_ceo_helpers.py | 6 +- factory/cli/run.py | 8 +-- .../workflow/contributed/design_v2/prompts.py | 37 ++---------- .../contributed/design_v2/workflow.py | 60 ++++++++++--------- 4 files changed, 44 insertions(+), 67 deletions(-) diff --git a/factory/cli/_ceo_helpers.py b/factory/cli/_ceo_helpers.py index 638e8ff4e..ef8ad25c8 100644 --- a/factory/cli/_ceo_helpers.py +++ b/factory/cli/_ceo_helpers.py @@ -194,12 +194,12 @@ def _validate_ceo_flags( just_plan: bool = getattr(args, "just_plan", False) if auto_approve and mode not in ("design", "design-v2"): - print("Error: --auto-approve only applies to --mode design", file=sys.stderr) + print("Error: --auto-approve only applies to --mode design or design-v2", file=sys.stderr) return 1 if just_plan: if mode not in ("design", "design-v2"): - print("Error: --just-plan requires --mode design", file=sys.stderr) + print("Error: --just-plan requires --mode design or design-v2", file=sys.stderr) return 1 if from_plan: print("Error: --just-plan and --from-plan are mutually exclusive.", file=sys.stderr) @@ -210,7 +210,7 @@ def _validate_ceo_flags( if from_plan: if mode not in ("design", "design-v2"): - print("Error: --from-plan requires --mode design", file=sys.stderr) + print("Error: --from-plan requires --mode design or design-v2", file=sys.stderr) return 1 if focus: print("Error: --from-plan and --focus are mutually exclusive.", file=sys.stderr) diff --git a/factory/cli/run.py b/factory/cli/run.py index 450158718..93dc83a62 100644 --- a/factory/cli/run.py +++ b/factory/cli/run.py @@ -405,8 +405,8 @@ def cmd_run(args: argparse.Namespace) -> int: mode = getattr(args, "mode", "auto") warn_deprecated_mode(mode) auto_approve: bool = getattr(args, "auto_approve", False) - if auto_approve and mode != "design": - print("Error: --auto-approve only applies to --mode design", file=sys.stderr) + if auto_approve and mode not in ("design", "design-v2"): + print("Error: --auto-approve only applies to --mode design or design-v2", file=sys.stderr) return 1 force_fresh = mode == "auto-fresh" if mode in ("auto", "auto-fresh"): @@ -430,7 +430,7 @@ def cmd_run(args: argparse.Namespace) -> int: file=sys.stderr, ) return 1 - if focus and mode not in ("design", "research"): + if focus and mode not in ("design", "design-v2", "research"): print( f"Error: --focus (targeted mode) only works in design or research mode, got '{mode}'. " "The project must already be built before targeting specific items.", @@ -455,7 +455,7 @@ def cmd_run(args: argparse.Namespace) -> int: print(f" Cleaned {len(pruned)} stale worktree(s)", file=sys.stderr) budget_kwargs = dict(min_growth=min_growth, max_new=max_new, branch=branch) - skip_improve = mode in ("design", "meta") or discover_only + skip_improve = mode in ("design", "design-v2", "meta") or discover_only overwrite = getattr(args, "overwrite", None) diff --git a/factory/workflow/contributed/design_v2/prompts.py b/factory/workflow/contributed/design_v2/prompts.py index 9751e695d..d4b2babc6 100644 --- a/factory/workflow/contributed/design_v2/prompts.py +++ b/factory/workflow/contributed/design_v2/prompts.py @@ -310,42 +310,13 @@ If a tester produced thin output, re-invoke it with a more specific prompt. +Each tester should output: Acceptance Criteria Verification (PASS/FAIL per +criterion with evidence), Edge Case Findings (steps to reproduce, expected +vs actual), and User Intent Verification (does output match user's ask). + Write a brief QA summary to the end of qa-plan.json noting which approaches completed and any quality issues.""" -# Used by the QA Director when spawning adversarial testers (referenced in QA_DIRECTOR_PROMPT). -ADVERSARIAL_PROMPT = """\ -You are an adversarial tester. Your job: break the implementation. - -Read: -- `.factory/strategy/current.md` — the design document and acceptance criteria -- `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for -- `.factory/reviews/builder-latest.md` — builder's work summary -- Source code changes (use git diff) - -Procedure: -1. Read the acceptance criteria from current.md -2. For each criterion, attempt to verify it by running the code -3. Try edge cases, invalid inputs, boundary conditions -4. Check that the implementation matches user intent, not just the plan -5. Look for security issues, error handling gaps, missing validations - -Output format: -# Adversarial Test Report - -## Acceptance Criteria Verification -- [ ] Criterion 1: PASS/FAIL — evidence -- [ ] Criterion 2: PASS/FAIL — evidence - -## Edge Case Findings -- Finding: - - Steps to reproduce: - - Expected: - - Actual: - -## User Intent Verification -- Does the output match what the user asked for? Evidence: <...>""" - GATE_QA_PROMPT = """\ You are the CEO reviewing QA results. This is the final gate before merge. diff --git a/factory/workflow/contributed/design_v2/workflow.py b/factory/workflow/contributed/design_v2/workflow.py index e4e918bfc..e020c0911 100644 --- a/factory/workflow/contributed/design_v2/workflow.py +++ b/factory/workflow/contributed/design_v2/workflow.py @@ -110,18 +110,24 @@ def workflow() -> Workflow: wf.nodes["init_user_intent"] = FnNode( id="init_user_intent", command=( - 'python3 -c "' - "import datetime, os; " - "project = '{project_path}'; " + "python3 -c \"" + "import datetime, os, sys; " + "from pathlib import Path; " + "project = sys.argv[1]; " + "intent = Path(f'{project}/.factory/strategy/user-intent.md'); " + "if intent.exists() and intent.stat().st_size > 0: " + " print('User intent ledger already exists, skipping'); sys.exit(0); " "ts = datetime.datetime.now().isoformat(timespec='seconds'); " - "idea = os.environ.get('FOCUS', os.environ.get('FACTORY_IDEA', " - "'No idea provided')); " - "content = f'# User Intent Ledger\\n\\n## [{ts}] Initial Idea\\n" - "{idea}\\n'; " - "open(f'{project}/.factory/strategy/user-intent.md', 'w').write(" - "content); " - "print(f'User intent ledger initialized at {ts}')" - '"' + "idea = os.environ.get('FACTORY_IDEA', ''); " + "if not idea: " + " bl = Path(f'{project}/.factory/strategy/backlog.md'); " + " idea = bl.read_text().strip().splitlines()[0] if bl.exists() and bl.stat().st_size > 0 else ''; " + "if not idea: idea = 'No idea provided'; " + "Path(f'{project}/.factory/strategy').mkdir(parents=True, exist_ok=True); " + "content = f'# User Intent Ledger\\\\n\\\\n## [{ts}] Initial Idea\\\\n{idea}\\\\n'; " + "intent.write_text(content); " + "print(f'User intent ledger initialized at {ts}')\" " + "\"{project_path}\"" ), writes={".factory/strategy/user-intent.md"}, notes="Creates the user intent ledger with the initial idea.", @@ -244,10 +250,10 @@ def workflow() -> Workflow: wf.nodes["synthesize_qa"] = FnNode( id="synthesize_qa", command=( - 'python3 -c "' + "python3 -c \"" "from pathlib import Path; " - "import re; " - "project = '{project_path}'; " + "import re, sys; " + "project = sys.argv[1]; " "reports = []; " "for p in sorted(Path(f'{project}/.factory/reviews').glob(" "'adversarial-*-latest.md')): " @@ -259,34 +265,34 @@ def workflow() -> Workflow: " for line in text.splitlines(): " " stripped = line.strip(); " " if stripped.startswith('- ') or stripped.startswith('* '): " - " key = re.sub(r'\\s+', ' ', " + " key = re.sub(r'\\\\s+', ' ', " "stripped[2:].strip().lower()[:80]); " " findings.setdefault(key, []).append(tester_slug); " "high = [(k, v) for k, v in findings.items() if len(v) >= 2]; " "medium = [(k, v) for k, v in findings.items() if len(v) == 1]; " - "out = ['# Synthesized QA Report\\n']; " + "out = ['# Synthesized QA Report\\\\n']; " "hc = Path(f'{project}/.factory/reviews/health-check.md'); " "cr = Path(f'{project}/.factory/reviews/code-review.md'); " - "out.append('## Health Check\\n'); " + "out.append('## Health Check\\\\n'); " "out.append(hc.read_text() if hc.exists() else '(not available)'); " - "out.append('\\n## Code Review\\n'); " + "out.append('\\\\n## Code Review\\\\n'); " "out.append(cr.read_text() if cr.exists() else '(not available)'); " - "out.append('\\n## High-Confidence Adversarial Findings " - "(caught by 2+ testers)\\n'); " + "out.append('\\\\n## High-Confidence Adversarial Findings " + "(caught by 2+ testers)\\\\n'); " "[out.append(f'- {k} (testers: {v})') for k, v in high]; " "if not high: out.append('- (none)'); " - "out.append('\\n## Medium-Confidence Adversarial Findings " - "(single tester)\\n'); " + "out.append('\\\\n## Medium-Confidence Adversarial Findings " + "(single tester)\\\\n'); " "[out.append(f'- {k} (tester: {v[0]})') for k, v in medium]; " "if not medium: out.append('- (none)'); " - "out.append('\\n## Raw Adversarial Reports\\n'); " - "[out.append(f'### Tester: {slug}\\n{text}\\n') " + "out.append('\\\\n## Raw Adversarial Reports\\\\n'); " + "[out.append(f'### Tester: {slug}\\\\n{text}\\\\n') " "for slug, text in reports]; " "Path(f'{project}/.factory/reviews/qa-synthesized.md').write_text(" - "'\\n'.join(out)); " + "'\\\\n'.join(out)); " "print(f'Synthesized {len(high)} high + {len(medium)} medium " - "findings from {len(reports)} adversarial reports')" - '"' + "findings from {len(reports)} adversarial reports')\" " + "\"{project_path}\"" ), reads={ ".factory/reviews/health-check.md", From 9227b5107f752295bf6153e9faaaf3e0cd299897 Mon Sep 17 00:00:00 2001 From: akashgit Date: Sat, 29 Aug 2026 17:05:56 -0400 Subject: [PATCH 7/9] fix: rewrite init_user_intent FnNode to avoid SyntaxError in python3 -c The command used compound if/else blocks after semicolons in a python3 -c one-liner, which Python does not allow. Rewritten using ternary expressions and or-chains: early exit via (sys.exit(0) if cond else None), idea fallback via env or backlog or default. Co-Authored-By: Claude Opus 4.6 (1M context) --- factory/workflow/contributed/design_v2/workflow.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/factory/workflow/contributed/design_v2/workflow.py b/factory/workflow/contributed/design_v2/workflow.py index e020c0911..ea055b779 100644 --- a/factory/workflow/contributed/design_v2/workflow.py +++ b/factory/workflow/contributed/design_v2/workflow.py @@ -115,14 +115,13 @@ def workflow() -> Workflow: "from pathlib import Path; " "project = sys.argv[1]; " "intent = Path(f'{project}/.factory/strategy/user-intent.md'); " - "if intent.exists() and intent.stat().st_size > 0: " - " print('User intent ledger already exists, skipping'); sys.exit(0); " + "(sys.exit(0) if intent.exists() and intent.stat().st_size > 0 else None); " "ts = datetime.datetime.now().isoformat(timespec='seconds'); " - "idea = os.environ.get('FACTORY_IDEA', ''); " - "if not idea: " - " bl = Path(f'{project}/.factory/strategy/backlog.md'); " - " idea = bl.read_text().strip().splitlines()[0] if bl.exists() and bl.stat().st_size > 0 else ''; " - "if not idea: idea = 'No idea provided'; " + "bl = Path(f'{project}/.factory/strategy/backlog.md'); " + "idea = os.environ.get('FACTORY_IDEA', '') " + "or (bl.read_text().strip().splitlines()[0] " + "if bl.exists() and bl.stat().st_size > 0 else '') " + "or 'No idea provided'; " "Path(f'{project}/.factory/strategy').mkdir(parents=True, exist_ok=True); " "content = f'# User Intent Ledger\\\\n\\\\n## [{ts}] Initial Idea\\\\n{idea}\\\\n'; " "intent.write_text(content); " From b8baaf1423dc3125f88d940f2d05c637bdc3d75a Mon Sep 17 00:00:00 2001 From: akashgit Date: Sat, 29 Aug 2026 17:09:09 -0400 Subject: [PATCH 8/9] feat: add mandatory code review phase to QA Director prompt Add PHASE 3 (CODE REVIEW) to QA_DIRECTOR_PROMPT in design_v2. The code review runs after all adversarial testers complete and is mandatory regardless of K or adversarial results. Critical/high-severity findings are flagged as blocking in the QA summary. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../workflow/contributed/design_v2/prompts.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/factory/workflow/contributed/design_v2/prompts.py b/factory/workflow/contributed/design_v2/prompts.py index d4b2babc6..f2270c9a4 100644 --- a/factory/workflow/contributed/design_v2/prompts.py +++ b/factory/workflow/contributed/design_v2/prompts.py @@ -244,7 +244,7 @@ - `.factory/strategy/user-intent.md` — what the user ACTUALLY asked for - `.factory/reviews/builder-latest.md` — what the builder implemented -Your task has TWO phases: +Your task has THREE phases: PHASE 1 — DESIGN QA APPROACHES Analyze the acceptance criteria, the user's intent, and the builder's @@ -270,6 +270,7 @@ - Simple implementation (3-5 criteria): 2 testers - Medium implementation (5-10 criteria): 3 testers - Complex implementation (10+ criteria, security, integrations): 4-5 testers +Note: the code review runs separately as Phase 3 and does NOT count toward K. For each approach, design a TAILORED prompt — not a generic template. Bad: "Test the implementation for edge cases" @@ -314,8 +315,23 @@ criterion with evidence), Edge Case Findings (steps to reproduce, expected vs actual), and User Intent Verification (does output match user's ask). +PHASE 3 — CODE REVIEW (MANDATORY) +After all adversarial testers complete, run a code review: +``` +factory agent code_reviewer --task "Review the code changes on this branch. \ +Use /code-review for a thorough review covering correctness bugs, security \ +issues, edge cases, missing tests, style, scope creep, and simplification \ +opportunities. Focus on the diff — what changed, not the entire codebase." \ +--project {project_path} +``` + +The code reviewer writes to `.factory/reviews/code-review.md`. +This step is MANDATORY — always run it regardless of K or adversarial results. +If the code review finds critical or high-severity issues, flag them in your +QA summary as blocking. + Write a brief QA summary to the end of qa-plan.json noting which -approaches completed and any quality issues.""" +approaches completed, code review results, and any quality issues.""" GATE_QA_PROMPT = """\ You are the CEO reviewing QA results. This is the final gate before merge. From 2ed581747fdcfcbdf4647bcdb1f5a94945b70696 Mon Sep 17 00:00:00 2001 From: akashgit Date: Sat, 29 Aug 2026 17:12:07 -0400 Subject: [PATCH 9/9] fix: run code review in parallel with adversarial testers, not sequentially The QA Director's Phase 2 now spawns K adversarial testers + 1 code reviewer all in parallel (backgrounded with &, then wait). Previously the code review was a separate Phase 3 that ran after testers completed. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../workflow/contributed/design_v2/prompts.py | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/factory/workflow/contributed/design_v2/prompts.py b/factory/workflow/contributed/design_v2/prompts.py index f2270c9a4..d1852174c 100644 --- a/factory/workflow/contributed/design_v2/prompts.py +++ b/factory/workflow/contributed/design_v2/prompts.py @@ -295,18 +295,35 @@ - One approach MUST verify user intent against user-intent.md - Prompts must reference specific acceptance criteria and implementation details -PHASE 2 — EXECUTE QA -For each approach in the plan, spawn an adversarial tester agent: +PHASE 2 — EXECUTE QA (ALL IN PARALLEL) +Spawn ALL of the following in parallel — K adversarial testers plus one +mandatory code reviewer: + +For each adversarial approach in the plan: +``` +factory agent adversarial_tester --review-tag --task "" --project {project_path} & ``` -factory agent adversarial_tester --task "" --project {project_path} + +Plus one mandatory code review (always, regardless of K): +``` +factory agent code_reviewer --task "Review the code changes on this branch. \ +Use /code-review for a thorough review covering correctness bugs, security \ +issues, edge cases, missing tests, style, scope creep, and simplification \ +opportunities. Focus on the diff — what changed, not the entire codebase." \ +--project {project_path} & ``` -Each tester writes to `.factory/reviews/adversarial--latest.md`. +Then `wait` for all K+1 agents to complete. + +Each adversarial tester writes to `.factory/reviews/adversarial--latest.md`. +The code reviewer writes to `.factory/reviews/code-review.md`. -After ALL testers complete, review quality: +After ALL agents complete, review quality: - Each adversarial report exists and has substantive findings - All acceptance criteria are covered by at least one tester - No tester missed its assigned focus area +- Code review completed — if it found critical or high-severity issues, + flag them in your QA summary as blocking - Critical findings are actually reproducible (spot-check) If a tester produced thin output, re-invoke it with a more specific prompt. @@ -315,21 +332,6 @@ criterion with evidence), Edge Case Findings (steps to reproduce, expected vs actual), and User Intent Verification (does output match user's ask). -PHASE 3 — CODE REVIEW (MANDATORY) -After all adversarial testers complete, run a code review: -``` -factory agent code_reviewer --task "Review the code changes on this branch. \ -Use /code-review for a thorough review covering correctness bugs, security \ -issues, edge cases, missing tests, style, scope creep, and simplification \ -opportunities. Focus on the diff — what changed, not the entire codebase." \ ---project {project_path} -``` - -The code reviewer writes to `.factory/reviews/code-review.md`. -This step is MANDATORY — always run it regardless of K or adversarial results. -If the code review finds critical or high-severity issues, flag them in your -QA summary as blocking. - Write a brief QA summary to the end of qa-plan.json noting which approaches completed, code review results, and any quality issues."""