diff --git a/skills/pm/SKILL.md b/skills/pm/SKILL.md index 6af401304..e920a11de 100644 --- a/skills/pm/SKILL.md +++ b/skills/pm/SKILL.md @@ -58,6 +58,11 @@ After every MCP response, do these three things: Print the MCP content text to the user first. +Tell users they do not need to invent speculative answers. If a question is +unknown, stakeholder-dependent, too broad, or safer to decide later, route it +through the existing assumptions / decide-later / deferred mechanisms instead +of presenting it as a confirmed requirement. + Then check: does `meta.ask_user_question` exist? - **YES** → Pass it directly to `AskUserQuestion`: @@ -70,7 +75,9 @@ Then check: does `meta.ask_user_question` exist? - If `meta.skip_eligible == true`: add a skip option based on `meta.classification`: - `classification == "decide_later"` → add option `{"label": "Decide later", "description": "Skip — will be recorded as an open item in the PRD"}` - `classification == "deferred"` → add option `{"label": "Defer to dev", "description": "Skip — this technical decision will be deferred to the development phase"}` - - Generate 2-3 suggested answers as the other options. + - Generate 2-3 suggested answers as the other options. Include a non-speculative + uncertainty option when appropriate, such as `Not sure yet — record as an + assumption or decide-later item`. **C. Relay answer back:** diff --git a/src/ouroboros/bigbang/pm_document.py b/src/ouroboros/bigbang/pm_document.py index 9df05b247..f5b0d9d4b 100644 --- a/src/ouroboros/bigbang/pm_document.py +++ b/src/ouroboros/bigbang/pm_document.py @@ -60,6 +60,8 @@ - Write for a PM audience — avoid technical jargon unless necessary - Be specific and actionable — avoid vague statements - Preserve all information from the interview — do not invent requirements +- Preserve uncertainty explicitly: keep tentative claims in Assumptions and + unresolved/stakeholder-dependent/unknown items in Decide Later - Use the interview conversation to add context and nuance beyond the \ structured seed data - Keep the tone professional but accessible diff --git a/src/ouroboros/bigbang/pm_interview.py b/src/ouroboros/bigbang/pm_interview.py index 52fa2eb68..1aab9f657 100644 --- a/src/ouroboros/bigbang/pm_interview.py +++ b/src/ouroboros/bigbang/pm_interview.py @@ -55,12 +55,22 @@ log = structlog.get_logger() +PM_UNCERTAINTY_GUIDANCE = ( + "If a product question is not settled, do not invent certainty. " + "Treat uncertain answers as explicit PM signal: record assumptions when " + "the user is making a tentative claim, or decide-later items when the " + "answer depends on missing information, a stakeholder decision, or a " + "future product choice." +) + _SEED_DIR = Path.home() / ".ouroboros" / "seeds" -_PM_SYSTEM_PROMPT_PREFIX = """\ +_PM_SYSTEM_PROMPT_PREFIX = f"""\ You are a Product Requirements interviewer helping a PM define their product. Focus on: goal, user stories, constraints, success criteria, assumptions. +{PM_UNCERTAINTY_GUIDANCE} + """ _OPENING_QUESTION = ( @@ -71,7 +81,10 @@ _EXTRACTION_SYSTEM_PROMPT = """\ You are a requirements extraction engine. Given a PM interview transcript, -extract structured product requirements. +extract structured product requirements. Preserve uncertainty explicitly: do not +turn uncertain, stakeholder-dependent, or unknown answers into confirmed +requirements. Put tentative claims in assumptions and unresolved choices in +decide_later_items. Respond ONLY with valid JSON in this exact format: { diff --git a/src/ouroboros/cli/commands/pm.py b/src/ouroboros/cli/commands/pm.py index 1bd576101..96be1f163 100644 --- a/src/ouroboros/cli/commands/pm.py +++ b/src/ouroboros/cli/commands/pm.py @@ -21,6 +21,7 @@ build_pm_completion_summary, maybe_complete_pm_interview, ) +from ouroboros.bigbang.pm_interview import PM_UNCERTAINTY_GUIDANCE from ouroboros.cli.formatters import console from ouroboros.cli.formatters.panels import print_error, print_info, print_success, print_warning from ouroboros.cli.formatters.prompting import multiline_prompt_async @@ -493,7 +494,8 @@ async def _run_pm_interview( print_success(f"Resumed session: {resume_id}") else: - # New session — ask the opening question first + # New session — show uncertainty guidance before the first PM answer + print_info(PM_UNCERTAINTY_GUIDANCE) opening = engine.get_opening_question() console.print(f"\n[bold yellow]?[/] {opening}\n") diff --git a/src/ouroboros/mcp/tools/pm_handler.py b/src/ouroboros/mcp/tools/pm_handler.py index 74f51f001..0b2555b86 100644 --- a/src/ouroboros/mcp/tools/pm_handler.py +++ b/src/ouroboros/mcp/tools/pm_handler.py @@ -36,7 +36,7 @@ maybe_complete_pm_interview, ) from ouroboros.bigbang.pm_document import save_pm_document -from ouroboros.bigbang.pm_interview import PMInterviewEngine +from ouroboros.bigbang.pm_interview import PM_UNCERTAINTY_GUIDANCE, PMInterviewEngine from ouroboros.config import get_clarification_model from ouroboros.core.initial_context import resolve_initial_context_input from ouroboros.core.types import Result @@ -846,7 +846,10 @@ async def _handle_start( ) # Build response text — include skip hint when applicable - start_text = f"PM interview started. Session ID: {state.interview_id}\n\n{question}" + start_text = ( + f"PM interview started. Session ID: {state.interview_id}\n\n" + f"{PM_UNCERTAINTY_GUIDANCE}\n\n{question}" + ) if is_decide_later: start_text += ( "\n\n💡 This question can be deferred. " diff --git a/tests/unit/bigbang/test_pm_document_generator.py b/tests/unit/bigbang/test_pm_document_generator.py index 0ecdf78a1..b1da04004 100644 --- a/tests/unit/bigbang/test_pm_document_generator.py +++ b/tests/unit/bigbang/test_pm_document_generator.py @@ -11,6 +11,7 @@ import pytest from ouroboros.bigbang.pm_document import ( + _PM_GENERATION_SYSTEM_PROMPT, PMDocumentGenerator, ) from ouroboros.bigbang.pm_seed import PMSeed, UserStory @@ -52,6 +53,14 @@ """ +def test_pm_generation_prompt_preserves_uncertainty_sections() -> None: + """LLM PM generation keeps uncertainty in existing Assumptions/Decide Later buckets.""" + assert "Preserve uncertainty explicitly" in _PM_GENERATION_SYSTEM_PROMPT + assert "Assumptions" in _PM_GENERATION_SYSTEM_PROMPT + assert "Decide Later" in _PM_GENERATION_SYSTEM_PROMPT + assert "unknown items" in _PM_GENERATION_SYSTEM_PROMPT + + def _make_seed(**overrides) -> PMSeed: """Create a PMSeed with sensible defaults for testing.""" defaults = { diff --git a/tests/unit/bigbang/test_pm_interview.py b/tests/unit/bigbang/test_pm_interview.py index 63245a99a..19d3c6a22 100644 --- a/tests/unit/bigbang/test_pm_interview.py +++ b/tests/unit/bigbang/test_pm_interview.py @@ -18,7 +18,12 @@ InterviewState, InterviewStatus, ) -from ouroboros.bigbang.pm_interview import PMInterviewEngine +from ouroboros.bigbang.pm_interview import ( + _EXTRACTION_SYSTEM_PROMPT, + _PM_SYSTEM_PROMPT_PREFIX, + PM_UNCERTAINTY_GUIDANCE, + PMInterviewEngine, +) from ouroboros.bigbang.pm_seed import PMSeed, UserStory from ouroboros.bigbang.question_classifier import ( ClassificationResult, @@ -34,6 +39,22 @@ ) +class TestPMUncertaintyGuidance: + """Regression coverage for PM uncertainty guidance (#1153).""" + + def test_pm_interviewer_prompt_tells_users_not_to_invent_certainty(self) -> None: + """PM interviews explicitly preserve uncertainty instead of forcing guesses.""" + assert PM_UNCERTAINTY_GUIDANCE in _PM_SYSTEM_PROMPT_PREFIX + assert "do not invent certainty" in _PM_SYSTEM_PROMPT_PREFIX + assert "decide-later items" in _PM_SYSTEM_PROMPT_PREFIX + + def test_pm_extraction_prompt_preserves_unknowns_as_unresolved(self) -> None: + """Extraction keeps unknown/stakeholder-dependent answers out of confirmed requirements.""" + assert "unknown answers" in _EXTRACTION_SYSTEM_PROMPT + assert "confirmed" in _EXTRACTION_SYSTEM_PROMPT + assert "decide_later_items" in _EXTRACTION_SYSTEM_PROMPT + + def _mock_completion(content: str = "What problem does this solve?") -> CompletionResponse: """Create a mock completion response.""" return CompletionResponse( @@ -1004,6 +1025,60 @@ async def test_generates_seed_from_interview(self, tmp_path: Path) -> None: assert len(seed.constraints) == 2 assert seed.interview_id == "test_001" + @pytest.mark.asyncio + async def test_uncertain_answer_is_preserved_as_assumption_and_decide_later( + self, tmp_path: Path + ) -> None: + """Uncertain PM answers can be recorded without becoming confirmed requirements.""" + adapter = _make_adapter() + engine = _make_engine(adapter, tmp_path) + + extraction_response = json.dumps( + { + "product_name": "StakeholderFlow", + "goal": "Capture product direction without fake certainty", + "user_stories": [], + "constraints": [], + "success_criteria": [], + "decide_later_items": ["Stakeholder needs to decide the launch metric"], + "assumptions": [ + "Team currently assumes weekly active use is the likely success signal" + ], + } + ) + adapter.complete = AsyncMock(return_value=Result.ok(_mock_completion(extraction_response))) + + state = InterviewState( + interview_id="test_uncertain", + initial_context="Plan a stakeholder-dependent product", + status=InterviewStatus.COMPLETED, + rounds=[ + InterviewRound( + round_number=1, + question="What success metric proves adoption?", + user_response=( + "I don't know yet; a stakeholder needs to decide. " + "For now, assume weekly active use might be the signal." + ), + ), + ], + ) + + result = await engine.generate_pm_seed(state) + + assert result.is_ok + seed = result.value + assert seed.user_stories == () + assert seed.decide_later_items == ("Stakeholder needs to decide the launch metric",) + assert seed.assumptions == ( + "Team currently assumes weekly active use is the likely success signal", + ) + + messages = adapter.complete.await_args.args[0] + assert messages[0].content == _EXTRACTION_SYSTEM_PROMPT + assert "turn uncertain, stakeholder-dependent" in messages[0].content + assert "I don't know yet; a stakeholder needs to decide" in messages[1].content + @pytest.mark.asyncio async def test_includes_deferred_items_in_decide_later(self, tmp_path: Path) -> None: """LLM-extracted deferred items are merged into decide_later_items on PMSeed.""" diff --git a/tests/unit/cli/test_pm_interactive_logging.py b/tests/unit/cli/test_pm_interactive_logging.py index 87c38483d..a6fb2fb29 100644 --- a/tests/unit/cli/test_pm_interactive_logging.py +++ b/tests/unit/cli/test_pm_interactive_logging.py @@ -10,6 +10,7 @@ import typer from ouroboros.bigbang.interview import InterviewRound +from ouroboros.bigbang.pm_interview import PM_UNCERTAINTY_GUIDANCE from ouroboros.cli.commands.pm import _run_pm_interview, pm_command from ouroboros.core.types import Result @@ -82,7 +83,7 @@ def fake_run(coro: object) -> None: async def test_run_pm_interview_new_session_uses_multiline_prompt_and_shows_progress( tmp_path: Path, ) -> None: - """New PM sessions should use the multiline prompt for both answers.""" + """New PM sessions should show uncertainty guidance before the first prompt.""" initial_state = _build_state() recorded_state = _build_state() recorded_state.is_complete = True @@ -98,6 +99,16 @@ async def test_run_pm_interview_new_session_uses_multiline_prompt_and_shows_prog engine.decide_later_items = [] engine.format_decide_later_summary.return_value = "" + info_messages: list[str] = [] + + async def fake_multiline_prompt(prompt: str) -> str: + if prompt == "Your response" and not fake_multiline_prompt.answers_used: + assert PM_UNCERTAINTY_GUIDANCE in info_messages + fake_multiline_prompt.answers_used += 1 + return ["Initial context", "Actual answer"][fake_multiline_prompt.answers_used - 1] + + fake_multiline_prompt.answers_used = 0 + with ( patch.object(Path, "home", return_value=tmp_path), patch("ouroboros.cli.commands.pm.create_llm_adapter", return_value=object()), @@ -108,9 +119,12 @@ async def test_run_pm_interview_new_session_uses_multiline_prompt_and_shows_prog patch("ouroboros.cli.commands.pm._save_cli_pm_meta"), patch( "ouroboros.cli.commands.pm.multiline_prompt_async", - side_effect=["Initial context", "Actual answer"], + side_effect=fake_multiline_prompt, ) as mock_prompt, - patch("ouroboros.cli.commands.pm.print_info") as mock_print_info, + patch( + "ouroboros.cli.commands.pm.print_info", + side_effect=info_messages.append, + ) as mock_print_info, patch("ouroboros.cli.commands.pm.print_success"), ): await _run_pm_interview(resume_id=None, model="test-model", backend="codex", debug=False) @@ -127,8 +141,10 @@ async def test_run_pm_interview_new_session_uses_multiline_prompt_and_shows_prog assert engine.save_state.await_args_list[-1].args[0] is recorded_state messages = [call.args[0] for call in mock_print_info.call_args_list] + assert PM_UNCERTAINTY_GUIDANCE in messages assert "Starting interview..." in messages assert "Generating next question..." in messages + assert messages.index(PM_UNCERTAINTY_GUIDANCE) < messages.index("Starting interview...") assert messages.index("Starting interview...") < messages.index("Generating next question...") diff --git a/tests/unit/mcp/tools/test_pm_handler.py b/tests/unit/mcp/tools/test_pm_handler.py index 16c7b000e..60533e618 100644 --- a/tests/unit/mcp/tools/test_pm_handler.py +++ b/tests/unit/mcp/tools/test_pm_handler.py @@ -18,6 +18,7 @@ from ouroboros.mcp.tools.pm_handler import ( _DATA_DIR, PMInterviewHandler, + PM_UNCERTAINTY_GUIDANCE, _check_completion, _compute_deferred_diff, _detect_action, @@ -629,6 +630,26 @@ def test_uses_custom_data_dir(self, tmp_path: Path) -> None: call_kwargs = mock_engine_cls.create.call_args assert call_kwargs.kwargs["state_dir"] == tmp_path + @pytest.mark.asyncio + async def test_start_response_places_uncertainty_guidance_before_question( + self, + tmp_path: Path, + ) -> None: + """MCP start response must show PM uncertainty guidance before question text.""" + engine = _make_engine_stub() + state = _make_state(interview_id="pm_guidance") + engine.ask_opening_and_start = AsyncMock(return_value=Result.ok(state)) + engine.ask_next_question = AsyncMock(return_value=Result.ok("What should we build first?")) + engine.save_state = AsyncMock(return_value=Result.ok(tmp_path / "pm_guidance.json")) + handler = PMInterviewHandler(pm_engine=engine, data_dir=tmp_path) + + result = await handler.handle({"initial_context": "Build a launch checklist"}) + + assert result.is_ok + text = result.value.text_content + assert PM_UNCERTAINTY_GUIDANCE in text + assert text.index(PM_UNCERTAINTY_GUIDANCE) < text.index("What should we build first?") + # ── Action auto-detection tests (AC 13) ───────────────────────