diff --git a/grimoire-runner/pyproject.toml b/grimoire-runner/pyproject.toml index c0c9418..315674e 100644 --- a/grimoire-runner/pyproject.toml +++ b/grimoire-runner/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "langchain-ollama>=0.1", "wyrdbound-dice>=0.0.1", "wyrdbound-rng>=0.0.1", + "jsonschema>=4.0", ] [project.optional-dependencies] diff --git a/grimoire-runner/src/grimoire_runner/core/loader.py b/grimoire-runner/src/grimoire_runner/core/loader.py index 1c19782..0b1eefb 100644 --- a/grimoire-runner/src/grimoire_runner/core/loader.py +++ b/grimoire-runner/src/grimoire_runner/core/loader.py @@ -13,6 +13,7 @@ FlowDefinition, InputDefinition, LLMSettingsDefinition, + LLMValidationDefinition, OutputDefinition, StepDefinition, StepType, @@ -392,6 +393,11 @@ def _parse_step_definition(self, data: dict[str, Any]) -> StepDefinition: if "llm_settings" in data: llm_settings = LLMSettingsDefinition(**data["llm_settings"]) + # Parse LLM validation + validation = None + if "validation" in data: + validation = LLMValidationDefinition(**data["validation"]) + return StepDefinition( id=data["id"], name=data.get("name"), # Make name optional @@ -415,6 +421,7 @@ def _parse_step_definition(self, data: dict[str, Any]) -> StepDefinition: prompt_id=data.get("prompt_id"), prompt_data=data.get("prompt_data", {}), llm_settings=llm_settings, + validation=validation, # Conditional step fields if_condition=data.get("if"), # Map 'if' to 'if_condition' then_actions=data.get("then"), diff --git a/grimoire-runner/src/grimoire_runner/executors/action_strategies.py b/grimoire-runner/src/grimoire_runner/executors/action_strategies.py index f7859aa..d458a55 100644 --- a/grimoire-runner/src/grimoire_runner/executors/action_strategies.py +++ b/grimoire-runner/src/grimoire_runner/executors/action_strategies.py @@ -55,9 +55,11 @@ def execute( resolved_value = value logger.debug(f"Action set_value: Using boolean directly for {path}") else: - # Resolve template in value for non-dict, non-boolean values - resolved_value = context.resolve_template(str(value)) - logger.debug(f"Action set_value: Resolved template for {path}") + # Check if this is a variable assignment and if we can preserve object types + resolved_value = self._resolve_value_with_type_preservation( + value, path, context, system + ) + logger.debug(f"Action set_value: Resolved value for {path}") # Use namespaced paths to avoid collision during flow execution current_namespace = context.get_current_flow_namespace() @@ -83,6 +85,99 @@ def execute( # Default to outputs context.set_output(path, resolved_value) + def _resolve_value_with_type_preservation( + self, + value: Any, + path: str, + context: "ExecutionContext", + system: "System | None" = None, + ) -> Any: + """Resolve template value while preserving object types for typed variables.""" + # Get the expected variable type if this is a variable assignment + expected_type = None + if path.startswith("variables.") and system: + variable_name = path[10:] # Remove "variables." prefix + expected_type = self._get_variable_type(variable_name, context, system) + logger.debug(f"Variable {variable_name} has expected type: {expected_type}") + + logger.debug( + f"Resolving value for {path}: {repr(value)} (type: {type(value).__name__})" + ) + + # First resolve the template to get the actual value + resolved_value = context.resolve_template(str(value)) + logger.debug( + f"Template resolved to: {repr(resolved_value)} (type: {type(resolved_value).__name__})" + ) + + # For roll_result type variables, try to preserve RollResult objects + if expected_type == "roll_result": + # Check if the resolved value is already a RollResult object + from ..models.roll_result import RollResult + + if isinstance(resolved_value, RollResult): + logger.debug(f"Template resolved to RollResult object for {path}") + return resolved_value + + # Post-processing: if we expected a roll_result but got a string, try to convert + if expected_type == "roll_result" and isinstance(resolved_value, str): + converted_result = self._convert_string_to_roll_result(resolved_value) + if converted_result: + logger.debug(f"Converted string to RollResult for {path}") + return converted_result + + return resolved_value + + def _get_variable_type( + self, + variable_name: str, + context: "ExecutionContext", + system: "System | None" = None, + ) -> str | None: + """Get the expected type for a variable from the flow definition.""" + if not system: + return None + + # Try to get the current flow definition to find variable type + current_execution = context.get_current_execution() + if current_execution and hasattr(current_execution, "flow_id"): + flow_id = current_execution.flow_id + if flow_id in system.flows: + flow_def = system.flows[flow_id] + for var_def in flow_def.variables: + if var_def.id == variable_name: + return var_def.type + return None + + def _convert_string_to_roll_result(self, value_str: str) -> Any: + """Try to convert a string back to a RollResult object if it looks like one.""" + from ..models.roll_result import RollResult + + # This is a simple heuristic - in practice, once we preserve the object properly, + # this shouldn't be needed, but it's here as a fallback + if not isinstance(value_str, str): + return None + + # Look for patterns like "15 = 15 (1d20: 15) + 0" which indicate a dice roll result + import re + + pattern = r"^(\d+)\s*=.*\(1d\d+.*\).*$" + match = re.match(pattern, value_str.strip()) + + if match: + try: + total = int(match.group(1)) + # Create a basic RollResult from the parsed information + return RollResult( + total=total, + detail=value_str, + expression="unknown", # We'd need more parsing to get this + ) + except (ValueError, AttributeError): + pass + + return None + class DisplayValueActionStrategy(ActionStrategy): """Strategy for handling display_value actions.""" @@ -123,7 +218,39 @@ def execute( """Execute a log_event action.""" event_type = action_data.get("type", "unknown") event_data = action_data.get("data", {}) - logger.debug(f"Event: {event_type} - {event_data}") + + # Resolve templates in event_data if it's a string + if isinstance(event_data, str): + resolved_event_data = context.resolve_template(event_data) + else: + resolved_event_data = event_data + + logger.debug(f"Event: {event_type} - {resolved_event_data}") + + +class LogMessageActionStrategy(ActionStrategy): + """Strategy for handling log_message actions.""" + + def get_action_type(self) -> str: + return "log_message" + + def execute( + self, + action_data: dict[str, Any], + context: "ExecutionContext", + system: "System | None" = None, + ) -> None: + """Execute a log_message action.""" + message = action_data.get("message", "") + + # Resolve templates in the message + resolved_message = context.resolve_template(str(message)) + + # Add the message to the execution context for UI display + context.add_action_message(f"📝 {resolved_message}") + + # Also log it for debugging + logger.debug(f"Action log_message: {resolved_message}") class SwapValuesActionStrategy(ActionStrategy): @@ -302,6 +429,7 @@ def _register_default_strategies(self) -> None: SetValueActionStrategy(), DisplayValueActionStrategy(), LogEventActionStrategy(), + LogMessageActionStrategy(), SwapValuesActionStrategy(), FlowCallActionStrategy(self.table_executor_factory), GetValueActionStrategy(), diff --git a/grimoire-runner/src/grimoire_runner/executors/dice_executor.py b/grimoire-runner/src/grimoire_runner/executors/dice_executor.py index e00ef03..fea08bf 100644 --- a/grimoire-runner/src/grimoire_runner/executors/dice_executor.py +++ b/grimoire-runner/src/grimoire_runner/executors/dice_executor.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any from ..integrations.dice_integration import DiceIntegration +from ..models.roll_result import RollResult from .base import BaseStepExecutor if TYPE_CHECKING: @@ -73,12 +74,21 @@ def _execute_dice_roll( else: logger.debug(f"Dice roll: {roll_expression} = {result.total}") - # Prepare result data + # Create structured RollResult object + roll_result = RollResult( + total=result.total, + detail=result.detailed_result or f"{result.total}", + expression=result.expression, + breakdown=result.breakdown, + individual_rolls=result.rolls, + ) + + # Prepare result data with both the structured object and legacy fields result_data = { - "result": result.total, - "expression": roll_expression, - "breakdown": result.breakdown if hasattr(result, "breakdown") else None, - "individual_rolls": result.rolls if hasattr(result, "rolls") else None, + "result": roll_result, # New structured result + "expression": result.expression, + "breakdown": result.breakdown, + "individual_rolls": result.rolls, } return StepResult( diff --git a/grimoire-runner/src/grimoire_runner/executors/llm_executor.py b/grimoire-runner/src/grimoire_runner/executors/llm_executor.py index b5afddb..1a5b645 100644 --- a/grimoire-runner/src/grimoire_runner/executors/llm_executor.py +++ b/grimoire-runner/src/grimoire_runner/executors/llm_executor.py @@ -1,8 +1,12 @@ """LLM generation step executor using LangChain.""" +import json import logging +import re from typing import TYPE_CHECKING +import jsonschema + from ..integrations.llm_integration import LLMIntegration from .base import BaseStepExecutor @@ -18,8 +22,219 @@ class LLMExecutor(BaseStepExecutor): """Executor for LLM generation steps.""" def __init__(self): + super().__init__() self.llm_integration = LLMIntegration() + def _extract_json_from_response(self, response: str) -> tuple[dict, bool]: + """ + Extract JSON from LLM response, handling various formats. + Returns (json_dict, success_flag) + """ + + # Try different extraction patterns + patterns = [ + # JSON in markdown code blocks + r"```json\s*\n(.*?)\n\s*```", + r"```\s*\n(\{.*?\})\s*\n```", + # JSON objects in text + r"\{[^{}]*\}", + ] + + for pattern in patterns: + matches = re.findall(pattern, response, re.DOTALL | re.MULTILINE) + for match in matches: + try: + parsed = json.loads(match.strip()) + return parsed, True + except json.JSONDecodeError: + continue + + # Try parsing the entire response as JSON + try: + parsed = json.loads(response.strip()) + return parsed, True + except json.JSONDecodeError: + pass + + return {}, False + + def _validate_json_schema(self, data: dict, schema: dict) -> tuple[bool, list[str]]: + """ + Validate JSON data against schema. + Returns (is_valid, error_messages) + """ + try: + jsonschema.validate(data, schema) + return True, [] + except jsonschema.ValidationError as e: + return False, [str(e)] + except Exception as e: + return False, [f"Validation error: {str(e)}"] + + def _create_cleanup_template( + self, raw_response: str, schema: dict, validation_errors: list[str] + ) -> str: + """Create a cleanup template for failed validation.""" + errors_text = "\n".join(f"- {error}" for error in validation_errors) + + template = f"""The following LLM response did not match the expected JSON schema: + +RESPONSE: +{raw_response} + +EXPECTED SCHEMA: +{schema} + +VALIDATION ERRORS: +{errors_text} + +Please provide ONLY the corrected JSON object with no additional text or formatting.""" + return template + + def _attempt_llm_call_with_validation( + self, prompt_template: str, prompt_data: dict, validation_config, settings: dict + ) -> tuple[dict, bool, int, str, list[str]]: + """ + Make LLM call with validation and automatic cleanup. + Returns (result_json, success, attempts, raw_response, errors) + """ + max_attempts = getattr(validation_config, "max_attempts", 3) + cleanup_enabled = getattr(validation_config, "cleanup_enabled", True) + + logger.debug( + f"Starting validation-aware LLM generation: max_attempts={max_attempts}, cleanup_enabled={cleanup_enabled}" + ) + + for attempt in range(max_attempts): + logger.debug(f"Attempt {attempt + 1}/{max_attempts}") + + # Make the LLM call + if attempt == 0: + current_template = prompt_template + else: + # Add guidance for retry attempts + current_template = f"{prompt_template}\n\nIMPORTANT: Please respond with valid JSON only, no additional text or formatting." + logger.debug("Retry attempt with modified template") + + # Log resolved prompt for this attempt (debug only) + from jinja2 import Template + + try: + template = Template(current_template) + resolved_prompt = template.render(**prompt_data) + logger.debug( + f"Resolved Prompt (attempt {attempt + 1}):\n{resolved_prompt}" + ) + except Exception as e: + logger.debug(f"Could not resolve prompt for logging: {e}") + + raw_response = self.llm_integration.generate_content( + prompt_template=current_template, context=prompt_data, **settings + ) + + logger.debug(f"Raw LLM Response (attempt {attempt + 1}):\n{raw_response}") + + # Extract JSON + json_data, json_extracted = self._extract_json_from_response(raw_response) + logger.debug(f"JSON extraction successful: {json_extracted}") + + if not json_extracted: + logger.debug("JSON extraction failed") + if attempt < max_attempts - 1 and cleanup_enabled: + # Try cleanup call + cleanup_template = """The following response could not be parsed as JSON: + +{{ raw_response }} + +Please provide the same information as a valid JSON object with no additional text or formatting.""" + + logger.debug("Attempting cleanup call for JSON extraction") + cleanup_response = self.llm_integration.generate_content( + prompt_template=cleanup_template, + context={"raw_response": raw_response}, + **settings, + ) + logger.debug(f"Cleanup response:\n{cleanup_response}") + + cleanup_json, cleanup_extracted = self._extract_json_from_response( + cleanup_response + ) + + if cleanup_extracted: + json_data = cleanup_json + raw_response = cleanup_response + logger.debug("Cleanup successful, JSON extracted") + else: + logger.debug("Cleanup failed, retrying") + continue # Try again with next attempt + else: + return ( + {}, + False, + attempt + 1, + raw_response, + ["Could not extract JSON from response"], + ) + + # Validate against schema if provided + if validation_config.type == "json_schema" and hasattr( + validation_config, "schema" + ): + logger.debug("Validating against JSON schema") + is_valid, errors = self._validate_json_schema( + json_data, validation_config.schema + ) + logger.debug(f"Schema validation result: valid={is_valid}") + + if not is_valid: + logger.debug(f"Schema validation errors: {errors}") + if attempt < max_attempts - 1 and cleanup_enabled: + # Try cleanup call with schema + cleanup_template = self._create_cleanup_template( + raw_response, validation_config.schema, errors + ) + logger.debug("Attempting cleanup call for schema validation") + cleanup_response = self.llm_integration.generate_content( + prompt_template=cleanup_template, context={}, **settings + ) + logger.debug(f"Schema cleanup response:\n{cleanup_response}") + + cleanup_json, cleanup_extracted = ( + self._extract_json_from_response(cleanup_response) + ) + + if cleanup_extracted: + cleanup_valid, cleanup_errors = self._validate_json_schema( + cleanup_json, validation_config.schema + ) + if cleanup_valid: + logger.debug("Schema cleanup successful") + return ( + cleanup_json, + True, + attempt + 1, + cleanup_response, + [], + ) + logger.debug("Schema cleanup failed, retrying") + continue # Try again with next attempt + else: + return json_data, False, attempt + 1, raw_response, errors + + # Success! + logger.debug(f"LLM generation successful on attempt {attempt + 1}") + return json_data, True, attempt + 1, raw_response, [] + + # All attempts failed + logger.debug(f"All {max_attempts} attempts failed") + return ( + {}, + False, + max_attempts, + raw_response, + ["Maximum validation attempts exceeded"], + ) + def execute( self, step: "StepDefinition", context: "ExecutionContext", system: "System" ) -> "StepResult": @@ -27,6 +242,8 @@ def execute( from ..models.flow import StepResult try: + logger.debug(f"=== Executing LLM Generation Step: {step.id} ===") + # Check if LLM is enabled/available if not self.llm_integration.is_available(): logger.warning(f"LLM not available for step {step.id}, skipping") @@ -42,6 +259,13 @@ def execute( }, ) + # Log provider and model information + logger.debug(f"LLM Provider: {self.llm_integration.provider}") + if hasattr(self.llm_integration._llm, "model"): + logger.debug(f"LLM Model: {self.llm_integration._llm.model}") + elif hasattr(self.llm_integration._llm, "model_name"): + logger.debug(f"LLM Model: {self.llm_integration._llm.model_name}") + # Get the prompt template prompt_template = self._get_prompt_template(step, system) if not prompt_template: @@ -51,27 +275,115 @@ def execute( error="No prompt template found for LLM generation", ) + # Log the template before resolution + logger.debug(f"Prompt Template (before resolution):\n{prompt_template}") + # Prepare prompt data prompt_data = {} if step.prompt_data: + logger.debug("Resolving template inputs:") for key, value in step.prompt_data.items(): - prompt_data[key] = context.resolve_template(str(value)) + resolved_value = context.resolve_template(str(value)) + prompt_data[key] = resolved_value + logger.debug(f" {key}: '{value}' -> '{resolved_value}'") + else: + logger.debug("No prompt_data defined for this step") # Get LLM settings settings = step.llm_settings or {} + if settings: + logger.debug(f"LLM Settings: {settings}") + else: + logger.debug("No LLM settings defined for this step") - # Generate content - generated_content = self.llm_integration.generate_content( - prompt_template=prompt_template, - context=prompt_data, - **(settings.__dict__ if hasattr(settings, "__dict__") else {}), - ) + # Check if validation is configured + if step.validation: + logger.debug( + f"Validation configured: type={step.validation.type}, max_attempts={step.validation.max_attempts}" + ) + + # Use validation-aware generation + result_json, success, attempts, raw_response, errors = ( + self._attempt_llm_call_with_validation( + prompt_template, + prompt_data, + step.validation, + settings.__dict__ if hasattr(settings, "__dict__") else {}, + ) + ) + + logger.debug( + f"Validation results: success={success}, attempts={attempts}" + ) + if not success: + logger.debug(f"Validation errors: {errors}") + + # Set context variables based on validation results + context.set_variable("llm_validation_successful", success) + context.set_variable("llm_validation_attempts", attempts) + context.set_variable("raw_llm_response", raw_response) + + if success: + context.set_variable("llm_result", result_json) + generated_content = result_json + logger.debug(f"Validation successful, result: {result_json}") + else: + context.set_variable("llm_validation_errors", errors) - logger.debug(f"LLM generation completed for step {step.id}") - logger.debug(f"Generated content: {generated_content[:100]}...") + # Handle failure based on configuration + on_failure = getattr(step.validation, "on_failure", "continue") + logger.debug( + f"Handling validation failure with strategy: {on_failure}" + ) - # Store result in context for actions and provide a common 'result' key - context.set_variable("llm_result", generated_content) + if on_failure == "fallback" and hasattr( + step.validation, "fallback_value" + ): + context.set_variable("llm_fallback_used", True) + context.set_variable( + "llm_result", step.validation.fallback_value + ) + generated_content = step.validation.fallback_value + logger.debug( + f"Using fallback value: {step.validation.fallback_value}" + ) + elif on_failure == "fail": + logger.debug("Failing step due to validation failure") + return StepResult( + step_id=step.id, + success=False, + error=f"LLM validation failed: {'; '.join(errors)}", + ) + else: # continue + context.set_variable("llm_fallback_used", False) + context.set_variable("llm_result", result_json) + generated_content = result_json + logger.debug(f"Continuing with invalid result: {result_json}") + else: + logger.debug("No validation configured, using direct LLM generation") + + # Log resolved prompt (debug only) + from jinja2 import Template + + try: + template = Template(prompt_template) + resolved_prompt = template.render(**prompt_data) + logger.debug(f"Resolved Prompt:\n{resolved_prompt}") + except Exception as e: + logger.debug(f"Could not resolve prompt for logging: {e}") + + # Use original non-validated generation + generated_content = self.llm_integration.generate_content( + prompt_template=prompt_template, + context=prompt_data, + **(settings.__dict__ if hasattr(settings, "__dict__") else {}), + ) + + # Log raw response + logger.debug(f"Raw LLM Response:\n{generated_content}") + + # Store result in context for actions and provide a common 'result' key + context.set_variable("llm_result", generated_content) return StepResult( step_id=step.id, @@ -96,17 +408,22 @@ def execute( def _get_prompt_template(self, step: "StepDefinition", system: "System") -> str: """Get the prompt template for the step.""" if step.prompt_id: - # Load from system prompts (TODO: implement prompt loading) - logger.warning( - f"Prompt loading from system not yet implemented: {step.prompt_id}" - ) - return f"Generate content based on the provided context. Prompt ID: {step.prompt_id}" + # Load from system prompts + prompt = system.get_prompt(step.prompt_id) + if prompt: + logger.debug(f"Loaded prompt '{prompt.name}' (ID: {step.prompt_id})") + return prompt.prompt_template + else: + logger.warning(f"Prompt not found in system: {step.prompt_id}") + return f"Generate content based on the provided context. Prompt ID: {step.prompt_id}" # Use the step prompt as template if step.prompt: + logger.debug("Using step-level prompt template") return step.prompt # Default template + logger.debug("Using default prompt template") return "Generate appropriate content based on the provided context." def can_execute(self, step: "StepDefinition") -> bool: diff --git a/grimoire-runner/src/grimoire_runner/models/__init__.py b/grimoire-runner/src/grimoire_runner/models/__init__.py index f872971..4291228 100644 --- a/grimoire-runner/src/grimoire_runner/models/__init__.py +++ b/grimoire-runner/src/grimoire_runner/models/__init__.py @@ -11,4 +11,5 @@ "SourceDefinition", "ModelDefinition", "ExecutionContext", + "RollResult", ] diff --git a/grimoire-runner/src/grimoire_runner/models/context_data.py b/grimoire-runner/src/grimoire_runner/models/context_data.py index 4679f96..9527599 100644 --- a/grimoire-runner/src/grimoire_runner/models/context_data.py +++ b/grimoire-runner/src/grimoire_runner/models/context_data.py @@ -53,6 +53,9 @@ class ExecutionContext: step_history: list[str] = field(default_factory=list) checkpoints: dict[str, Checkpoint] = field(default_factory=dict) + # Action messages for UI display + action_messages: list[str] = field(default_factory=list) + # Template resolution (delegated to specialized resolver) template_resolver: TemplateResolver = field(default_factory=TemplateResolver) @@ -385,6 +388,16 @@ def get_state_snapshot(self) -> dict[str, Any]: "timestamp": datetime.now().isoformat(), } + def add_action_message(self, message: str) -> None: + """Add an action message to be displayed by the UI.""" + self.action_messages.append(message) + + def get_and_clear_action_messages(self) -> list[str]: + """Get all action messages and clear the list.""" + messages = self.action_messages.copy() + self.action_messages.clear() + return messages + def initialize_model_observables( self, model_definition, instance_id: str = None ) -> None: diff --git a/grimoire-runner/src/grimoire_runner/models/flow.py b/grimoire-runner/src/grimoire_runner/models/flow.py index ad954ba..b967495 100644 --- a/grimoire-runner/src/grimoire_runner/models/flow.py +++ b/grimoire-runner/src/grimoire_runner/models/flow.py @@ -100,6 +100,18 @@ class LLMSettingsDefinition: temperature: float = 0.7 +@dataclass +class LLMValidationDefinition: + """LLM response validation configuration.""" + + type: str # "json" or "json_schema" + schema: dict[str, Any] | None = None # JSON Schema for validation + max_attempts: int = 3 # Total attempts (original + retries) + cleanup_enabled: bool = True # Enable automatic cleanup + on_failure: str = "continue" # "continue", "fail", or "fallback" + fallback_value: Any = None # Value to use when validation fails + + @dataclass class StepDefinition: """Base flow step definition.""" @@ -137,6 +149,7 @@ class StepDefinition: prompt_id: str | None = None prompt_data: dict[str, Any] = field(default_factory=dict) llm_settings: LLMSettingsDefinition | None = None + validation: LLMValidationDefinition | None = None # For conditional if_condition: str | None = None # Condition to evaluate (alias for condition) diff --git a/grimoire-runner/src/grimoire_runner/models/roll_result.py b/grimoire-runner/src/grimoire_runner/models/roll_result.py new file mode 100644 index 0000000..259d608 --- /dev/null +++ b/grimoire-runner/src/grimoire_runner/models/roll_result.py @@ -0,0 +1,82 @@ +"""Roll result models for structured dice roll data.""" + +from dataclasses import dataclass +from typing import Any + + +@dataclass +class RollResult: + """Structured result from a dice roll operation.""" + + total: int + """Final numeric result of the roll.""" + + detail: str + """Detailed string representation of the roll (e.g., '11 = 11 (1d20: 11) + 0').""" + + expression: str + """The dice expression that was rolled (e.g., '1d20 + 0').""" + + # Optional fields for advanced dice information + breakdown: dict[str, Any] | None = None + """Detailed breakdown of the roll from wyrdbound-dice.""" + + individual_rolls: list | None = None + """List of individual die results.""" + + @property + def description(self) -> str: + """Alias for detail to support legacy template usage.""" + return self.detail + + def __str__(self) -> str: + """String representation shows the detailed result.""" + return self.detail + + def __int__(self) -> int: + """Integer representation returns the total.""" + return self.total + + def __lt__(self, other) -> bool: + """Less than comparison based on total.""" + if isinstance(other, int | float): + return self.total < other + elif isinstance(other, RollResult): + return self.total < other.total + return NotImplemented + + def __le__(self, other) -> bool: + """Less than or equal comparison based on total.""" + if isinstance(other, int | float): + return self.total <= other + elif isinstance(other, RollResult): + return self.total <= other.total + return NotImplemented + + def __gt__(self, other) -> bool: + """Greater than comparison based on total.""" + if isinstance(other, int | float): + return self.total > other + elif isinstance(other, RollResult): + return self.total > other.total + return NotImplemented + + def __ge__(self, other) -> bool: + """Greater than or equal comparison based on total.""" + if isinstance(other, int | float): + return self.total >= other + elif isinstance(other, RollResult): + return self.total >= other.total + return NotImplemented + + def __eq__(self, other) -> bool: + """Equality comparison based on total.""" + if isinstance(other, int | float): + return self.total == other + elif isinstance(other, RollResult): + return self.total == other.total + return NotImplemented + + def __ne__(self, other) -> bool: + """Inequality comparison based on total.""" + return not self.__eq__(other) diff --git a/grimoire-runner/src/grimoire_runner/models/step.py b/grimoire-runner/src/grimoire_runner/models/step.py index 2fcd672..d18cf2f 100644 --- a/grimoire-runner/src/grimoire_runner/models/step.py +++ b/grimoire-runner/src/grimoire_runner/models/step.py @@ -5,6 +5,7 @@ ChoiceDefinition, DiceSequenceDefinition, LLMSettingsDefinition, + LLMValidationDefinition, StepDefinition, StepResult, StepType, @@ -20,4 +21,5 @@ "TableRollDefinition", "DiceSequenceDefinition", "LLMSettingsDefinition", + "LLMValidationDefinition", ] diff --git a/grimoire-runner/src/grimoire_runner/services/template_service.py b/grimoire-runner/src/grimoire_runner/services/template_service.py index ba978e4..923e7ff 100644 --- a/grimoire-runner/src/grimoire_runner/services/template_service.py +++ b/grimoire-runner/src/grimoire_runner/services/template_service.py @@ -76,9 +76,39 @@ def resolve_template(self, template_str: str, context_data: Any) -> Any: f"Template: '{template_str}'" ) + # Enhance context with roll_result attribute access + enhanced_context = self._enhance_context_for_objects(context_data) + + # Check for simple variable reference that should preserve object type + template_str_stripped = template_str.strip() + if ( + template_str_stripped.startswith("{{") + and template_str_stripped.endswith("}}") + and template_str_stripped.count("{{") == 1 + and template_str_stripped.count("}}") == 1 + ): + # Extract variable name from {{ variable_name }} + var_content = template_str_stripped[2:-2].strip() + if "." not in var_content and " " not in var_content: + # Simple variable reference like {{ result }} + if var_content in enhanced_context: + original_obj = enhanced_context[var_content] + # If we have the original object, return it to preserve type + if ( + isinstance(original_obj, dict) + and "_original" in original_obj + ): + logger.debug( + f"Preserving original object type for variable: {var_content}" + ) + return original_obj["_original"] + # For other simple objects, return as-is + logger.debug(f"Returning simple variable as-is: {var_content}") + return original_obj + # NO FALLBACKS - template resolution must be explicit about missing variables # This will cause Jinja2 to raise UndefinedError for missing variables - result = template.render(context_data) + result = template.render(enhanced_context) # Try to parse as structured data if it looks like it parsed_result = self._try_parse_structured_data(result) @@ -96,6 +126,42 @@ def resolve_template(self, template_str: str, context_data: Any) -> Any: logger.error(error_msg) raise RuntimeError(error_msg) from e + def _enhance_context_for_objects(self, context_data: dict) -> dict: + """Enhance context to provide better object attribute access.""" + enhanced_context = context_data.copy() + + # Make RollResult objects more accessible in templates + from ..models.roll_result import RollResult + + def make_object_accessible(obj): + """Convert objects to be more accessible in Jinja2 templates.""" + if isinstance(obj, RollResult): + # Convert RollResult to a dict-like object that preserves all attributes + return { + "total": obj.total, + "detail": obj.detail, + "description": obj.description, # Alias for detail + "expression": obj.expression, + "breakdown": obj.breakdown, + "individual_rolls": obj.individual_rolls, + # Also preserve the original object for methods + "_original": obj, + } + elif isinstance(obj, dict): + # Recursively process dict values + return {k: make_object_accessible(v) for k, v in obj.items()} + elif isinstance(obj, list): + # Recursively process list items + return [make_object_accessible(item) for item in obj] + else: + return obj + + # Process all context values + for key, value in enhanced_context.items(): + enhanced_context[key] = make_object_accessible(value) + + return enhanced_context + def is_template(self, text: str) -> bool: """Check if a string contains template syntax.""" if not isinstance(text, str): @@ -159,10 +225,21 @@ def _try_parse_structured_data(self, result: str) -> Any: pass # Special handling for single-line YAML that might be intentional structured data - # but not simple display text like "Strength: +2" + # but not simple display text like "Strength: +2" or log messages if ":" in result and ( result.count("\n") > 0 - or not any(char in result for char in ["+", "-", "Strength", "Dexterity"]) + and not any( + phrase in result.lower() + for phrase in [ + "saving throw", + "justification", + "ability", + "type", + "roll", + "dice", + "damage", + ] + ) ): try: parsed = yaml.safe_load(result) diff --git a/grimoire-runner/src/grimoire_runner/ui/cli.py b/grimoire-runner/src/grimoire_runner/ui/cli.py index b26170d..e1828f1 100644 --- a/grimoire-runner/src/grimoire_runner/ui/cli.py +++ b/grimoire-runner/src/grimoire_runner/ui/cli.py @@ -136,6 +136,15 @@ def execute( logging.basicConfig( level=logging.DEBUG, format="%(levelname)s:%(name)s:%(message)s" ) + # Suppress noisy HTTP logs that provide little value for GRIMOIRE debugging + logging.getLogger("httpcore.http11").setLevel(logging.WARNING) + logging.getLogger("httpcore.connection").setLevel(logging.WARNING) + logging.getLogger("httpcore._backends.sync").setLevel(logging.WARNING) + logging.getLogger("httpx").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("requests").setLevel(logging.WARNING) + # Also suppress httpcore logs in general + logging.getLogger("httpcore").setLevel(logging.WARNING) elif verbose: logging.basicConfig( level=logging.INFO, format="%(levelname)s:%(name)s:%(message)s" diff --git a/grimoire-runner/src/grimoire_runner/ui/rich_tui.py b/grimoire-runner/src/grimoire_runner/ui/rich_tui.py index 121ee31..b9fc42d 100644 --- a/grimoire-runner/src/grimoire_runner/ui/rich_tui.py +++ b/grimoire-runner/src/grimoire_runner/ui/rich_tui.py @@ -555,6 +555,11 @@ def _execute_single_step(self, step, step_num: int) -> tuple[bool, str | None]: # Show step result with custom or generic messages if step_result.success: + # Display any action messages that were generated during step execution + action_messages = self.context.get_and_clear_action_messages() + for action_message in action_messages: + self._print_indented(f"[yellow]{action_message}[/yellow]") + # Check if step has a custom result message if step.result_message: # Resolve any templates in the custom message @@ -568,10 +573,21 @@ def _execute_single_step(self, step, step_num: int) -> tuple[bool, str | None]: f"Template resolution context - outputs.saving_throw_result: {self.context.outputs.get('saving_throw_result', 'NOT_FOUND')}" ) - resolved_message = ( - template_service.resolve_template_with_execution_context( - step.result_message, self.context, self.system - ) + # Create a temporary context that includes step result data for template resolution + # This makes the 'result' variable available in result_message templates + temp_context_data = { + "inputs": self.context.inputs, + "variables": self.context.variables, + "outputs": self.context.outputs, + "system": self.system, + } + + # Add step result data if available + if step_result.data: + temp_context_data.update(step_result.data) + + resolved_message = template_service.resolve_template( + step.result_message, temp_context_data ) logger.debug( @@ -724,18 +740,18 @@ def _execute_flow_call_step(self, step, step_num: int) -> tuple[bool, str | None # Store the result in the main context for template resolution self.context.outputs["result"] = sub_flow_outputs - self._print_indented( + self.console.print( f"[green]🔗 Sub-flow '{sub_flow_name}' completed successfully[/green]" ) if sub_flow_outputs: self._print_indented( - f"[dim] 📤 Outputs: {list(sub_flow_outputs.keys())}[/dim]" + f"[dim]📤 Outputs: {list(sub_flow_outputs.keys())}[/dim]" ) # Execute any flow_call step actions with result context if hasattr(step, "actions") and step.actions: self._print_indented( - "[cyan] ⚙️ Executing flow_call step actions...[/cyan]" + "[cyan]⚙️ Executing flow_call step actions...[/cyan]" ) # Create result object for template resolution @@ -755,11 +771,9 @@ def __init__(self, outputs): self.engine.action_executor.execute_actions( step.actions, self.context, {}, self.system ) - self._print_indented( - "[green] ✅ Step actions completed[/green]" - ) + self._print_indented("[green]✅ Step actions completed[/green]") except Exception as e: - self._print_indented(f"[red] ❌ Step action failed: {e}[/red]") + self._print_indented(f"[red]❌ Step action failed: {e}[/red]") return False, None # Create step result diff --git a/grimoire-runner/tests/systems/flow_test/flows/dice-flow.yaml b/grimoire-runner/tests/systems/flow_test/flows/dice-flow.yaml index 3dc7c37..e081ff9 100644 --- a/grimoire-runner/tests/systems/flow_test/flows/dice-flow.yaml +++ b/grimoire-runner/tests/systems/flow_test/flows/dice-flow.yaml @@ -5,13 +5,17 @@ name: "Dice Rolling Flow" description: "Flow demonstrating dice step types" variables: - dice_result: 0 - ability_scores: {} + - type: int + id: dice_result + description: "Result of the dice roll (just the total)" + - type: list + id: ability_scores + description: "List of ability scores" steps: - id: "simple-roll" name: "Simple Dice Roll" - type: "dice_roll" + type: dice_roll prompt: "Rolling a simple d6" roll: "1d6" output: "dice_result" @@ -19,7 +23,7 @@ steps: - type: "set_value" data: path: "variables.dice_result" - value: "{{ result }}" + value: "{{ result.total }}" next_step: "sequence-roll" - id: "sequence-roll" diff --git a/grimoire-runner/tests/systems/flow_test/flows/test_level_2_flow.yaml b/grimoire-runner/tests/systems/flow_test/flows/test_level_2_flow.yaml index 7ef470f..2a05f2e 100644 --- a/grimoire-runner/tests/systems/flow_test/flows/test_level_2_flow.yaml +++ b/grimoire-runner/tests/systems/flow_test/flows/test_level_2_flow.yaml @@ -11,7 +11,9 @@ outputs: type: text variables: - level_2_dice: 0 + - type: roll_result + id: level_2_dice + description: "Result of the level 2 dice roll" steps: - id: level_2_step diff --git a/grimoire-runner/tests/systems/flow_test/flows/test_level_3_flow.yaml b/grimoire-runner/tests/systems/flow_test/flows/test_level_3_flow.yaml index 7267e3a..cdc956e 100644 --- a/grimoire-runner/tests/systems/flow_test/flows/test_level_3_flow.yaml +++ b/grimoire-runner/tests/systems/flow_test/flows/test_level_3_flow.yaml @@ -11,7 +11,9 @@ outputs: type: text variables: - deep_dice_result: 0 + - type: roll_result + id: deep_dice_result + description: "Result of the deep dice roll" steps: - id: level_3_dice diff --git a/grimoire-runner/tests/systems/llm_validation_test/fixtures/flows/test_json_validation/basic.yaml b/grimoire-runner/tests/systems/llm_validation_test/fixtures/flows/test_json_validation/basic.yaml new file mode 100644 index 0000000..f6a2e97 --- /dev/null +++ b/grimoire-runner/tests/systems/llm_validation_test/fixtures/flows/test_json_validation/basic.yaml @@ -0,0 +1 @@ +action_description: "The character attempts to dodge a falling boulder while climbing a cliff" diff --git a/grimoire-runner/tests/systems/llm_validation_test/fixtures/flows/test_log_event_template_resolution/basic.yaml b/grimoire-runner/tests/systems/llm_validation_test/fixtures/flows/test_log_event_template_resolution/basic.yaml new file mode 100644 index 0000000..149fe9e --- /dev/null +++ b/grimoire-runner/tests/systems/llm_validation_test/fixtures/flows/test_log_event_template_resolution/basic.yaml @@ -0,0 +1 @@ +test_input: "Hello from test input" diff --git a/grimoire-runner/tests/systems/llm_validation_test/fixtures/flows/test_schema_validation/basic.yaml b/grimoire-runner/tests/systems/llm_validation_test/fixtures/flows/test_schema_validation/basic.yaml new file mode 100644 index 0000000..126d1b8 --- /dev/null +++ b/grimoire-runner/tests/systems/llm_validation_test/fixtures/flows/test_schema_validation/basic.yaml @@ -0,0 +1 @@ +action_description: "The character tries to resist a mind control spell" diff --git a/grimoire-runner/tests/systems/llm_validation_test/fixtures/flows/test_validation_fallback/basic.yaml b/grimoire-runner/tests/systems/llm_validation_test/fixtures/flows/test_validation_fallback/basic.yaml new file mode 100644 index 0000000..e7c47fa --- /dev/null +++ b/grimoire-runner/tests/systems/llm_validation_test/fixtures/flows/test_validation_fallback/basic.yaml @@ -0,0 +1 @@ +action_description: "The character jumps across a wide chasm" diff --git a/grimoire-runner/tests/systems/llm_validation_test/flows/test_json_validation.yaml b/grimoire-runner/tests/systems/llm_validation_test/flows/test_json_validation.yaml new file mode 100644 index 0000000..f22d8b0 --- /dev/null +++ b/grimoire-runner/tests/systems/llm_validation_test/flows/test_json_validation.yaml @@ -0,0 +1,40 @@ +id: test_json_validation +type: flow +name: "Test JSON Validation" +description: "Test flow for LLM JSON validation" +version: "1.0" + +inputs: + - type: str + id: action_description + description: "Description of an action requiring a saving throw" + +outputs: + - type: str + id: saving_throw_ability + description: "The ability required for the saving throw" + - type: str + id: reason + description: "Reason for the saving throw" + +steps: + - id: determine_save_ability_json + name: "Determine Save Ability (JSON validation)" + type: llm_generation + prompt: | + Based on the following action, determine which D&D ability score should be used for a saving throw. + + Action: {{ action_description }} + + Respond with a JSON object containing the ability and reason. + prompt_data: + action_description: "{{ inputs.action_description }}" + validation: + type: "json" + actions: + - set_value: + path: "outputs.saving_throw_ability" + value: "{{ llm_result.ability if llm_result.ability is defined else 'unknown' }}" + - set_value: + path: "outputs.reason" + value: "{{ llm_result.reason if llm_result.reason is defined else 'No reason provided' }}" diff --git a/grimoire-runner/tests/systems/llm_validation_test/flows/test_log_event_template_resolution.yaml b/grimoire-runner/tests/systems/llm_validation_test/flows/test_log_event_template_resolution.yaml new file mode 100644 index 0000000..66a7a3d --- /dev/null +++ b/grimoire-runner/tests/systems/llm_validation_test/flows/test_log_event_template_resolution.yaml @@ -0,0 +1,41 @@ +id: test_log_event_template_resolution +type: flow +name: "Test Log Event Template Resolution" +description: "Test that log_event actions resolve templates correctly" +version: "1.0" + +inputs: + - type: str + id: test_input + description: "Test input for template resolution" + +outputs: + - type: str + id: test_output + description: "Test output" + +variables: + - type: str + id: test_variable + description: "Test variable" + +steps: + - id: test_llm_with_log_event + name: "Test LLM Generation with Log Event" + type: llm_generation + prompt: | + Return a simple JSON object with a test message using the input: {{ test_input }} + prompt_data: + test_input: "{{ inputs.test_input }}" + validation: + type: "json" + actions: + - set_value: + path: "variables.test_variable" + value: "{{ llm_result.message if llm_result.message is defined else 'no message' }}" + - set_value: + path: "outputs.test_output" + value: "{{ variables.test_variable }}" + - log_event: + type: "test_event" + data: "Input: {{ inputs.test_input }}, Variable: {{ variables.test_variable }}, LLM Result: {{ llm_result }}, Raw Response: {{ raw_llm_response }}" diff --git a/grimoire-runner/tests/systems/llm_validation_test/flows/test_schema_validation.yaml b/grimoire-runner/tests/systems/llm_validation_test/flows/test_schema_validation.yaml new file mode 100644 index 0000000..a3ca4a3 --- /dev/null +++ b/grimoire-runner/tests/systems/llm_validation_test/flows/test_schema_validation.yaml @@ -0,0 +1,66 @@ +id: test_schema_validation +type: flow +name: "Test Schema Validation" +description: "Test flow for LLM JSON schema validation" +version: "1.0" + +inputs: + - type: str + id: action_description + description: "Description of an action requiring a saving throw" + +outputs: + - type: str + id: saving_throw_ability + description: "The ability required for the saving throw" + - type: str + id: reason + description: "Reason for the saving throw" + - type: bool + id: validation_successful + description: "Whether validation was successful" + +steps: + - id: determine_save_ability_schema + name: "Determine Save Ability (Schema validation)" + type: llm_generation + prompt: | + Based on the following action, determine which D&D ability score should be used for a saving throw. + + Action: {{ action_description }} + + Respond with a JSON object containing: + - ability: one of "strength", "dexterity", "constitution", "intelligence", "wisdom", "charisma" + - reason: a descriptive reason (at least 10 characters) + prompt_data: + action_description: "{{ inputs.action_description }}" + validation: + type: "json_schema" + schema: + type: "object" + properties: + ability: + type: "string" + enum: + [ + "strength", + "dexterity", + "constitution", + "intelligence", + "wisdom", + "charisma", + ] + reason: + type: "string" + minLength: 10 + required: ["ability", "reason"] + actions: + - set_value: + path: "outputs.saving_throw_ability" + value: "{{ llm_result.ability }}" + - set_value: + path: "outputs.reason" + value: "{{ llm_result.reason }}" + - set_value: + path: "outputs.validation_successful" + value: "{{ llm_validation_successful }}" diff --git a/grimoire-runner/tests/systems/llm_validation_test/flows/test_validation_fallback.yaml b/grimoire-runner/tests/systems/llm_validation_test/flows/test_validation_fallback.yaml new file mode 100644 index 0000000..7424401 --- /dev/null +++ b/grimoire-runner/tests/systems/llm_validation_test/flows/test_validation_fallback.yaml @@ -0,0 +1,69 @@ +id: test_validation_fallback +type: flow +name: "Test Validation with Fallback" +description: "Test flow for LLM validation with fallback behavior" +version: "1.0" + +inputs: + - type: str + id: action_description + description: "Description of an action requiring a saving throw" + +outputs: + - type: str + id: saving_throw_ability + description: "The ability required for the saving throw" + - type: str + id: reason + description: "Reason for the saving throw" + - type: bool + id: fallback_used + description: "Whether fallback value was used" + +steps: + - id: determine_save_ability_fallback + name: "Determine Save Ability (with fallback)" + type: llm_generation + prompt: | + Based on the following action, determine which D&D ability score should be used for a saving throw. + + Action: {{ action_description }} + + This prompt is intentionally designed to potentially fail validation for testing purposes. + Sometimes respond with invalid JSON or missing required fields. + prompt_data: + action_description: "{{ inputs.action_description }}" + validation: + type: "json_schema" + schema: + type: "object" + properties: + ability: + type: "string" + enum: + [ + "strength", + "dexterity", + "constitution", + "intelligence", + "wisdom", + "charisma", + ] + reason: + type: "string" + minLength: 10 + required: ["ability", "reason"] + on_failure: "fallback" + fallback_value: + ability: "dexterity" + reason: "Default fallback ability" + actions: + - set_value: + path: "outputs.saving_throw_ability" + value: "{{ llm_result.ability }}" + - set_value: + path: "outputs.reason" + value: "{{ llm_result.reason }}" + - set_value: + path: "outputs.fallback_used" + value: "{{ llm_fallback_used if llm_fallback_used is defined else false }}" diff --git a/grimoire-runner/tests/systems/llm_validation_test/system.yaml b/grimoire-runner/tests/systems/llm_validation_test/system.yaml new file mode 100644 index 0000000..9bc0dca --- /dev/null +++ b/grimoire-runner/tests/systems/llm_validation_test/system.yaml @@ -0,0 +1,7 @@ +id: llm_validation_test +name: "LLM Validation Test System" +description: "Test system for LLM validation functionality" +version: "1.0.0" + +settings: + dice_notation: "standard" diff --git a/grimoire-runner/tests/test_llm_validation.py b/grimoire-runner/tests/test_llm_validation.py new file mode 100644 index 0000000..daf8cc6 --- /dev/null +++ b/grimoire-runner/tests/test_llm_validation.py @@ -0,0 +1,200 @@ +"""Tests for LLM validation functionality.""" + +import sys +from unittest.mock import MagicMock, patch + +# Add src to path for imports +sys.path.append("src") + +from grimoire_runner.executors.llm_executor import LLMExecutor +from grimoire_runner.integrations.llm_integration import LLMIntegration +from grimoire_runner.models.step import LLMValidationDefinition + + +def create_mock_json_response(prompt_template, context, **kwargs): + """Create a mock JSON response for testing validation.""" + if "JSON" in prompt_template or "json" in prompt_template: + # Return a valid JSON response for most cases + return '{"ability": "dexterity", "reason": "dodging the falling boulder"}' + else: + # Return a mixed text response that needs JSON extraction + return """Based on the action, the appropriate saving throw would be: + +```json +{"ability": "dexterity", "reason": "dodging requires quick reflexes"} +``` + +This makes sense because dodging falling objects primarily requires agility.""" + + +def create_mock_invalid_json_response(prompt_template, context, **kwargs): + """Create a mock invalid JSON response for testing cleanup.""" + if "valid JSON object" in prompt_template or "corrected JSON" in prompt_template: + # Cleanup call - return valid JSON + return '{"ability": "strength", "reason": "lifting heavy objects"}' + else: + # Initial call - return invalid JSON + return 'This is not valid JSON: {ability: "strength" reason: missing comma}' + + +class TestLLMValidation: + """Test LLM validation functionality.""" + + @patch.object( + LLMIntegration, "generate_content", side_effect=create_mock_json_response + ) + def test_json_validation_success(self, mock_generate): + """Test successful JSON validation.""" + executor = LLMExecutor() + + # Create a step with JSON validation + step = MagicMock() + step.validation = LLMValidationDefinition(type="json") + step.prompt = "Generate JSON for a saving throw" + step.prompt_data = {"action": "dodge boulder"} + step.llm_settings = MagicMock() + step.llm_settings.__dict__ = {} + + # Test the validation method directly + result_json, success, attempts, raw_response, errors = ( + executor._attempt_llm_call_with_validation( + "Generate JSON response", {}, step.validation, {} + ) + ) + + assert success is True + assert attempts == 1 + assert "ability" in result_json + assert "reason" in result_json + assert result_json["ability"] == "dexterity" + assert len(errors) == 0 + + @patch.object( + LLMIntegration, + "generate_content", + side_effect=create_mock_invalid_json_response, + ) + def test_json_validation_with_cleanup(self, mock_generate): + """Test JSON validation with automatic cleanup.""" + executor = LLMExecutor() + + # Create a step with JSON validation + step = MagicMock() + step.validation = LLMValidationDefinition(type="json") + + # Test the validation method + result_json, success, attempts, raw_response, errors = ( + executor._attempt_llm_call_with_validation( + "Generate JSON response", {}, step.validation, {} + ) + ) + + # Should succeed after cleanup + assert success is True + assert attempts >= 1 + assert "ability" in result_json + assert "reason" in result_json + + # Verify cleanup was called + assert mock_generate.call_count >= 2 # Initial call + cleanup call + + def test_json_schema_validation_success(self): + """Test successful JSON schema validation.""" + executor = LLMExecutor() + + # Test data that matches schema + test_data = {"ability": "dexterity", "reason": "quick reflexes needed"} + schema = { + "type": "object", + "properties": { + "ability": { + "type": "string", + "enum": [ + "strength", + "dexterity", + "constitution", + "intelligence", + "wisdom", + "charisma", + ], + }, + "reason": {"type": "string", "minLength": 10}, + }, + "required": ["ability", "reason"], + } + + is_valid, errors = executor._validate_json_schema(test_data, schema) + + assert is_valid is True + assert len(errors) == 0 + + def test_json_schema_validation_failure(self): + """Test JSON schema validation failure.""" + executor = LLMExecutor() + + # Test data that doesn't match schema + test_data = {"ability": "invalid_ability", "reason": "short"} + schema = { + "type": "object", + "properties": { + "ability": { + "type": "string", + "enum": [ + "strength", + "dexterity", + "constitution", + "intelligence", + "wisdom", + "charisma", + ], + }, + "reason": {"type": "string", "minLength": 10}, + }, + "required": ["ability", "reason"], + } + + is_valid, errors = executor._validate_json_schema(test_data, schema) + + assert is_valid is False + assert len(errors) > 0 + + def test_json_extraction_from_markdown(self): + """Test JSON extraction from markdown code blocks.""" + executor = LLMExecutor() + + response = """Here's the result: + +```json +{"ability": "wisdom", "reason": "resisting mental influence"} +``` + +This should work for wisdom saving throws.""" + + json_data, success = executor._extract_json_from_response(response) + + assert success is True + assert json_data["ability"] == "wisdom" + assert json_data["reason"] == "resisting mental influence" + + def test_json_extraction_from_mixed_text(self): + """Test JSON extraction from mixed text.""" + executor = LLMExecutor() + + response = 'The saving throw should use {"ability": "constitution", "reason": "enduring physical stress"} based on the action.' + + json_data, success = executor._extract_json_from_response(response) + + assert success is True + assert json_data["ability"] == "constitution" + assert json_data["reason"] == "enduring physical stress" + + def test_json_extraction_failure(self): + """Test JSON extraction failure.""" + executor = LLMExecutor() + + response = "This is just plain text with no JSON whatsoever." + + json_data, success = executor._extract_json_from_response(response) + + assert success is False + assert json_data == {} diff --git a/grimoire-runner/tests/test_log_event_template_resolution.py b/grimoire-runner/tests/test_log_event_template_resolution.py new file mode 100644 index 0000000..8296cb0 --- /dev/null +++ b/grimoire-runner/tests/test_log_event_template_resolution.py @@ -0,0 +1,78 @@ +"""Test log_event template resolution.""" + +import logging +import sys +from unittest.mock import MagicMock, patch + +# Add src to path for imports +sys.path.append("src") + +from grimoire_runner.executors.action_strategies import LogEventActionStrategy + + +def test_log_event_template_resolution(): + """Test that log_event action resolves templates correctly.""" + + # Create a mock execution context + context = MagicMock() + context.resolve_template.return_value = ( + "Input: Hello World, Variable: test_value, Result: success" + ) + + # Create the action strategy + strategy = LogEventActionStrategy() + + # Mock the logger to capture the output + with patch.object( + logging.getLogger("grimoire_runner.executors.action_strategies"), "debug" + ) as mock_logger: + # Execute the action with template strings + action_data = { + "type": "test_event", + "data": "Input: {{ inputs.test_input }}, Variable: {{ variables.test_var }}, Result: {{ result }}", + } + + strategy.execute(action_data, context) + + # Verify template resolution was called + context.resolve_template.assert_called_once_with( + "Input: {{ inputs.test_input }}, Variable: {{ variables.test_var }}, Result: {{ result }}" + ) + + # Verify the logger was called with the resolved template + mock_logger.assert_called_once_with( + "Event: test_event - Input: Hello World, Variable: test_value, Result: success" + ) + + +def test_log_event_non_string_data(): + """Test that log_event action handles non-string data without template resolution.""" + + # Create a mock execution context + context = MagicMock() + + # Create the action strategy + strategy = LogEventActionStrategy() + + # Mock the logger to capture the output + with patch.object( + logging.getLogger("grimoire_runner.executors.action_strategies"), "debug" + ) as mock_logger: + # Execute the action with dict data (should not be template resolved) + action_data = {"type": "test_event", "data": {"key": "value", "number": 42}} + + strategy.execute(action_data, context) + + # Verify template resolution was NOT called for dict data + context.resolve_template.assert_not_called() + + # Verify the logger was called with the original dict + mock_logger.assert_called_once_with( + "Event: test_event - {'key': 'value', 'number': 42}" + ) + + +if __name__ == "__main__": + test_log_event_template_resolution() + test_log_event_non_string_data() + print("All log_event template resolution tests passed!") diff --git a/grimoire-runner/tests/test_log_message_formatting.py b/grimoire-runner/tests/test_log_message_formatting.py new file mode 100644 index 0000000..4fc7713 --- /dev/null +++ b/grimoire-runner/tests/test_log_message_formatting.py @@ -0,0 +1,151 @@ +"""Test log message formatting consistency.""" + +from grimoire_runner.executors.action_strategies import LogMessageActionStrategy +from grimoire_runner.models.context_data import ExecutionContext + + +class TestLogMessageFormatting: + """Test cases for log message action formatting consistency.""" + + def setup_method(self): + """Set up test fixtures.""" + self.strategy = LogMessageActionStrategy() + self.context = ExecutionContext() + + # Set up some test data in context + self.context.set_variable("saving_throw_ability", "dexterity") + self.context.set_variable("saving_throw_type", "basic") + + # Mock LLM result data by setting it as a variable (how it would be in actual execution) + self.context.set_variable( + "llm_result", + { + "ability": "dexterity", + "reason": "Dexterity is needed for quick reactions and balance to avoid falling debris.", + "type": "basic", + }, + ) + + def test_multiple_log_messages_consistent_formatting(self): + """Test that multiple log_message actions format consistently as plain text.""" + # Execute first log message action + action1_data = { + "message": "Saving Throw Ability: {{ variables.saving_throw_ability }}" + } + + self.strategy.execute(action1_data, self.context) + + # Execute second log message action + action2_data = {"message": "Justification: {{ variables.llm_result.reason }}"} + + self.strategy.execute(action2_data, self.context) + + # Get the action messages that were added + action_messages = self.context.get_and_clear_action_messages() + + # Should have two messages + assert len(action_messages) == 2 + + # Both messages should be formatted as plain text with 📝 emoji + expected_message1 = "📝 Saving Throw Ability: dexterity" + expected_message2 = "📝 Justification: Dexterity is needed for quick reactions and balance to avoid falling debris." + + assert action_messages[0] == expected_message1 + assert action_messages[1] == expected_message2 + + # Neither message should contain dictionary-like formatting + for message in action_messages: + assert not message.startswith("📝 {") + assert not message.endswith("}") + assert "': '" not in message + + def test_log_message_with_colon_not_parsed_as_yaml(self): + """Test that log messages with colons are not parsed as YAML structures.""" + # Test various messages that contain colons but should remain as plain text + test_cases = [ + ( + "Saving Throw Ability: {{ variables.saving_throw_ability }}", + "📝 Saving Throw Ability: dexterity", + ), + ( + "Saving Throw Type: {{ variables.saving_throw_type }}", + "📝 Saving Throw Type: basic", + ), + ("Roll Result: Success", "📝 Roll Result: Success"), + ("Damage Type: fire", "📝 Damage Type: fire"), + ("Dice Roll: 1d20+2", "📝 Dice Roll: 1d20+2"), + ] + + for input_message, expected_output in test_cases: + # Clear any previous messages + self.context.get_and_clear_action_messages() + + action_data = {"message": input_message} + self.strategy.execute(action_data, self.context) + + messages = self.context.get_and_clear_action_messages() + assert len(messages) == 1 + assert messages[0] == expected_output + + def test_log_message_template_resolution_preserves_text_format(self): + """Test that template resolution in log messages preserves text format.""" + # Set up more complex template data + self.context.set_variable("character_name", "Thorgar") + self.context.set_variable("spell_name", "Magic Missile") + + test_cases = [ + { + "message": "Character Name: {{ variables.character_name }}", + "expected": "📝 Character Name: Thorgar", + }, + { + "message": "Casting Spell: {{ variables.spell_name }}", + "expected": "📝 Casting Spell: Magic Missile", + }, + { + "message": "Action: {{ variables.character_name }} casts {{ variables.spell_name }}", + "expected": "📝 Action: Thorgar casts Magic Missile", + }, + ] + + for test_case in test_cases: + # Clear previous messages + self.context.get_and_clear_action_messages() + + action_data = {"message": test_case["message"]} + self.strategy.execute(action_data, self.context) + + messages = self.context.get_and_clear_action_messages() + assert len(messages) == 1 + assert messages[0] == test_case["expected"] + + def test_log_message_does_not_parse_gaming_terms_as_yaml(self): + """Test that gaming-specific terms in log messages don't get parsed as YAML.""" + # These should all be treated as plain text, not parsed as structured data + gaming_messages = [ + "Ability Score: Strength", + "Saving Throw: Constitution", + "Dice Roll: 2d6+3", + "Damage Roll: 1d8+2", + "Initiative Roll: 1d20+1", + "Attack Roll: Natural 20", + "Spell Level: 3rd level", + "Armor Class: 18", + ] + + for message in gaming_messages: + # Clear previous messages + self.context.get_and_clear_action_messages() + + action_data = {"message": message} + self.strategy.execute(action_data, self.context) + + messages = self.context.get_and_clear_action_messages() + assert len(messages) == 1 + + # Should be plain text with emoji prefix + expected = f"📝 {message}" + assert messages[0] == expected + + # Should not be parsed as a dictionary + assert not messages[0].startswith("📝 {") diff --git a/grimoire-runner/tests/test_nested_flow_execution.py b/grimoire-runner/tests/test_nested_flow_execution.py index 5473763..e59af00 100644 --- a/grimoire-runner/tests/test_nested_flow_execution.py +++ b/grimoire-runner/tests/test_nested_flow_execution.py @@ -75,8 +75,10 @@ def test_level_2_flow_executes_independently(self, engine, test_system): # Should also have the dice result in variables assert "level_2_dice" in result.variables dice_result = result.variables["level_2_dice"] - assert isinstance(dice_result, int) - assert 1 <= dice_result <= 6 # 1d6 range + from grimoire_runner.models.roll_result import RollResult + + assert isinstance(dice_result, RollResult) + assert 1 <= dice_result.total <= 6 # 1d6 range def test_level_3_flow_executes_independently(self, engine, test_system): """Test that level 3 flow can execute independently.""" @@ -98,8 +100,10 @@ def test_level_3_flow_executes_independently(self, engine, test_system): # Should have dice roll result in variables assert "deep_dice_result" in result.variables dice_result = result.variables["deep_dice_result"] - assert isinstance(dice_result, int) - assert 1 <= dice_result <= 4 # 1d4 range + from grimoire_runner.models.roll_result import RollResult + + assert isinstance(dice_result, RollResult) + assert 1 <= dice_result.total <= 4 # 1d4 range # Should also have the output assert "level_3_result" in result.outputs @@ -245,8 +249,10 @@ def test_flow_call_step_has_result_variable(self, engine, test_system): # Should have the dice result in variables assert "deep_dice_result" in result.variables dice_result = result.variables["deep_dice_result"] - assert isinstance(dice_result, int) - assert 1 <= dice_result <= 4 # 1d4 range + from grimoire_runner.models.roll_result import RollResult + + assert isinstance(dice_result, RollResult) + assert 1 <= dice_result.total <= 4 # 1d4 range # The output should contain the dice roll result (templated from {{ variables.deep_dice_result }}) # This tests that the result variable mechanism is working diff --git a/grimoire-runner/tests/test_template_yaml_regression.py b/grimoire-runner/tests/test_template_yaml_regression.py new file mode 100644 index 0000000..130a00e --- /dev/null +++ b/grimoire-runner/tests/test_template_yaml_regression.py @@ -0,0 +1,130 @@ +"""Test template service YAML parsing behavior for log messages.""" + +from grimoire_runner.services.template_service import TemplateService + + +class TestTemplateServiceYAMLParsing: + """Test cases for template service YAML parsing behavior.""" + + def setup_method(self): + """Set up test fixtures.""" + self.template_service = TemplateService() + + def test_template_service_does_not_parse_log_messages_as_yaml(self): + """Test that template service doesn't parse log-like messages as YAML.""" + # Test context that would cause the original bug + context = { + "variables": {"saving_throw_ability": "dexterity"}, + "inputs": {}, + "outputs": {}, + } + + # Template that would resolve to a string with colon + template_str = "Saving Throw Ability: {{ variables.saving_throw_ability }}" + + # Resolve the template + result = self.template_service.resolve_template( + template_str, context, "runtime" + ) + + # Should return plain string, not a dict + expected_result = "Saving Throw Ability: dexterity" + assert result == expected_result + assert isinstance(result, str) + assert not isinstance(result, dict) + + def test_template_service_gaming_terms_not_parsed_as_yaml(self): + """Test various gaming-related templates that should not be parsed as YAML.""" + context = { + "variables": { + "ability": "strength", + "dice_result": "1d20+2", + "damage_type": "fire", + "spell_level": "3rd", + }, + "inputs": {}, + "outputs": {}, + } + + test_cases = [ + ("Ability: {{ variables.ability }}", "Ability: strength"), + ("Dice Roll: {{ variables.dice_result }}", "Dice Roll: 1d20+2"), + ("Damage Type: {{ variables.damage_type }}", "Damage Type: fire"), + ("Spell Level: {{ variables.spell_level }}", "Spell Level: 3rd"), + ("Saving Throw Type: basic", "Saving Throw Type: basic"), + ("Roll Result: Success", "Roll Result: Success"), + ] + + for template_str, expected in test_cases: + result = self.template_service.resolve_template( + template_str, context, "runtime" + ) + assert result == expected, ( + f"Template '{template_str}' should resolve to '{expected}', got '{result}'" + ) + assert isinstance(result, str), ( + f"Result should be string, got {type(result)}" + ) + + def test_template_service_preserves_legitimate_yaml_parsing(self): + """Test that legitimate YAML structures are still parsed correctly.""" + context = { + "variables": {"data": "value1\nkey2: value2"}, + "inputs": {}, + "outputs": {}, + } + + # Multi-line YAML should still be parsed + template_str = "{{ variables.data }}" + result = self.template_service.resolve_template( + template_str, context, "runtime" + ) + + # This should parse as YAML because it's multi-line + # (Though this particular case might not parse as valid YAML, the point is + # that multi-line content should go through YAML parsing logic) + # The exact result depends on YAML parsing, but it should at least try + + # For this test, let's use clear valid YAML + context["variables"]["yaml_data"] = "key1: value1\nkey2: value2" + template_str = "{{ variables.yaml_data }}" + result = self.template_service.resolve_template( + template_str, context, "runtime" + ) + + # Should either be the original string or parsed as dict, but not treated as a single-line log message + assert result == "key1: value1\nkey2: value2" or result == { + "key1": "value1", + "key2": "value2", + } + + def test_regression_original_bug_scenario(self): + """Test the exact scenario that caused the original bug.""" + # This reproduces the context that was causing the inconsistent formatting + context = { + "variables": {"saving_throw_ability": "dexterity"}, + "inputs": {}, + "outputs": {}, + } + + # First log message template (was being parsed as YAML) + template1 = "Saving Throw Ability: {{ variables.saving_throw_ability }}" + result1 = self.template_service.resolve_template(template1, context, "runtime") + + # Second log message template (was working correctly) + context["variables"]["reason"] = ( + "Dexterity is needed for quick reactions and balance to avoid falling debris." + ) + template2 = "Justification: {{ variables.reason }}" + result2 = self.template_service.resolve_template(template2, context, "runtime") + + # Both should be strings, not dictionaries + assert result1 == "Saving Throw Ability: dexterity" + assert ( + result2 + == "Justification: Dexterity is needed for quick reactions and balance to avoid falling debris." + ) + assert isinstance(result1, str) + assert isinstance(result2, str) + assert not isinstance(result1, dict) + assert not isinstance(result2, dict) diff --git a/systems/knave_1e/fixtures/flows/perform_saving_throw/basic.yaml b/systems/knave_1e/fixtures/flows/perform_saving_throw/basic.yaml new file mode 100644 index 0000000..331a33a --- /dev/null +++ b/systems/knave_1e/fixtures/flows/perform_saving_throw/basic.yaml @@ -0,0 +1,14 @@ +actor: + abilities: + strength: { bonus: 1 } + dexterity: { bonus: 2 } + constitution: { bonus: 3 } + intelligence: { bonus: 4 } + wisdom: { bonus: 5 } + charisma: { bonus: 6 } +context_summary: |- + The player is trapped within a crumbling, subterranean temple + dedicated to a forgotten deity. As he cautiously proceeds through a + narrow corridor, a sudden, violent tremor shakes the temple. Dust + and debris rain down from the ceiling. They should roll a save for + whether they are able to avoid the falling debris. diff --git a/systems/knave_1e/fixtures/flows/roll_saving_throw/advantage.yaml b/systems/knave_1e/fixtures/flows/roll_saving_throw/advantage.yaml index cdff5c4..dcfdade 100644 --- a/systems/knave_1e/fixtures/flows/roll_saving_throw/advantage.yaml +++ b/systems/knave_1e/fixtures/flows/roll_saving_throw/advantage.yaml @@ -1,4 +1,3 @@ saving_throw_type: advantage -saving_throw_target: player saving_throw_modifier: 5 saving_throw_dc: 15 diff --git a/systems/knave_1e/fixtures/flows/roll_saving_throw/basic.yaml b/systems/knave_1e/fixtures/flows/roll_saving_throw/basic.yaml index f2d5708..8c8c507 100644 --- a/systems/knave_1e/fixtures/flows/roll_saving_throw/basic.yaml +++ b/systems/knave_1e/fixtures/flows/roll_saving_throw/basic.yaml @@ -1,4 +1,3 @@ saving_throw_type: basic -saving_throw_target: player saving_throw_modifier: 0 saving_throw_dc: 15 diff --git a/systems/knave_1e/fixtures/flows/roll_saving_throw/disadvantage.yaml b/systems/knave_1e/fixtures/flows/roll_saving_throw/disadvantage.yaml index 4ce46ea..355029b 100644 --- a/systems/knave_1e/fixtures/flows/roll_saving_throw/disadvantage.yaml +++ b/systems/knave_1e/fixtures/flows/roll_saving_throw/disadvantage.yaml @@ -1,4 +1,3 @@ saving_throw_type: disadvantage -saving_throw_target: player saving_throw_modifier: 2 saving_throw_dc: 12 diff --git a/systems/knave_1e/fixtures/flows/roll_saving_throw/failure.yaml b/systems/knave_1e/fixtures/flows/roll_saving_throw/failure.yaml index 653ddfd..8bbed0f 100644 --- a/systems/knave_1e/fixtures/flows/roll_saving_throw/failure.yaml +++ b/systems/knave_1e/fixtures/flows/roll_saving_throw/failure.yaml @@ -1,4 +1,3 @@ -saving_throw_type: "basic" -saving_throw_target: "player" +saving_throw_type: basic saving_throw_modifier: 0 saving_throw_dc: 20 # High DC to make failure certain diff --git a/systems/knave_1e/fixtures/flows/roll_saving_throw/success.yaml b/systems/knave_1e/fixtures/flows/roll_saving_throw/success.yaml index a19f0f8..2fcb727 100644 --- a/systems/knave_1e/fixtures/flows/roll_saving_throw/success.yaml +++ b/systems/knave_1e/fixtures/flows/roll_saving_throw/success.yaml @@ -1,4 +1,3 @@ -saving_throw_type: "basic" -saving_throw_target: "player" +saving_throw_type: basic saving_throw_modifier: 0 saving_throw_dc: 0 # Low DC to make success certain diff --git a/systems/knave_1e/flows/add_item_to_character.yaml b/systems/knave_1e/flows/add_item_to_character.yaml index b23a4bc..66fc9d8 100644 --- a/systems/knave_1e/flows/add_item_to_character.yaml +++ b/systems/knave_1e/flows/add_item_to_character.yaml @@ -19,7 +19,7 @@ outputs: id: "character" description: "The updated character with the new item" -variables: {} +variables: [] steps: - id: "add_item" diff --git a/systems/knave_1e/flows/character_creation.yaml b/systems/knave_1e/flows/character_creation.yaml index c8dc452..ae81513 100644 --- a/systems/knave_1e/flows/character_creation.yaml +++ b/systems/knave_1e/flows/character_creation.yaml @@ -1,22 +1,24 @@ -id: "character_creation" -type: "flow" +id: character_creation +type: flow name: "Character Creation" description: "Character creation process for {{ system.name }}" version: "1.0" inputs: [] outputs: - - type: "character" - id: "knave" + - type: character + id: knave validate: true variables: - hp_dice_roll: "" + - type: str + id: hp_dice_roll + description: "The dice roll used to determine hit points." steps: - - id: "roll_abilities" + - id: roll_abilities name: "Roll Abilities" - type: "dice_sequence" + type: dice_sequence prompt: "Roll 3d6 for each ability. The lowest die becomes your bonus." sequence: items: @@ -35,21 +37,21 @@ steps: value: "{{ result }}" display_as: "{{ item|title }}: {{ result }} (bonus +{{ result }}, defense {{ result + 10 }})" - - id: "ability_swap_choice" + - id: ability_swap_choice name: "Optional Ability Swap" - type: "player_choice" + type: player_choice prompt: "You may optionally swap the scores of two abilities." choices: - - id: "no_swap" + - id: no_swap label: "Keep abilities as rolled" next_step: "hit_points_choice" - - id: "swap_abilities" + - id: swap_abilities label: "Swap two ability scores" - next_step: "swap_values_step" + next_step: swap_values_step - - id: "swap_values_step" + - id: swap_values_step name: "Choose Abilities to Swap" - type: "player_choice" + type: player_choice prompt: "Choose two abilities to swap:" choice_source: table_from_values: "outputs.knave.abilities" @@ -59,21 +61,21 @@ steps: - swap_values: path1: "outputs.knave.abilities.{{ selected_items[0] }}.bonus" path2: "outputs.knave.abilities.{{ selected_items[1] }}.bonus" - next_step: "hit_points_choice" + next_step: hit_points_choice - - id: "hit_points_choice" + - id: hit_points_choice name: "Hit Points Rolling Method" - type: "player_choice" + type: player_choice prompt: "Choose how to roll your starting hit points:" choices: - - id: "standard_roll" + - id: standard_roll label: "Standard roll (1d8)" actions: - set_value: path: "variables.hp_dice_roll" value: "1d8" next_step: "roll_hit_points" - - id: "house_rule" + - id: house_rule label: "House rule: reroll results below 5" actions: - set_value: @@ -81,186 +83,186 @@ steps: value: "1d8r<5" next_step: "roll_hit_points" - - id: "roll_hit_points" + - id: roll_hit_points name: "Roll Hit Points" - type: "dice_roll" + type: dice_roll prompt: "Rolling for hit points..." roll: "{{ variables.hp_dice_roll }}" actions: - set_value: path: "outputs.knave.hit_points.max" - value: "{{ result }}" + value: "{{ result.total }}" - set_value: path: "outputs.knave.hit_points.current" - value: "{{ result }}" - next_step: "choose_weapon" + value: "{{ result.total }}" + next_step: choose_weapon - - id: "choose_weapon" + - id: choose_weapon name: "Choose Starting Weapon" - type: "player_choice" + type: player_choice prompt: "Choose one weapon to start with:" choice_source: - table: "weapons" + table: weapons actions: - flow_call: - flow: "add_item_to_character" + flow: add_item_to_character inputs: character: "outputs.knave" item: "{{ result }}" - next_step: "roll_starting_gear" + next_step: roll_starting_gear - - id: "roll_starting_gear" + - id: roll_starting_gear name: "Determine Starting Equipment" - type: "table_roll" + type: table_roll prompt: "Rolling for your starting equipment..." parallel: true tables: - - table: "armor" + - table: armor actions: - flow_call: - flow: "add_item_to_character" + flow: add_item_to_character inputs: character: "outputs.knave" item: "{{ result }}" - - table: "helmets_and_shields" + - table: helmets_and_shields actions: - flow_call: - flow: "add_item_to_character" + flow: add_item_to_character inputs: character: "outputs.knave" item: "{{ result }}" - - table: "dungeoneering_gear" + - table: dungeoneering_gear count: 2 actions: - flow_call: - flow: "add_items_to_character" + flow: add_items_to_character inputs: character: "outputs.knave" items: "{{ results }}" - - table: "general_gear_1" + - table: general_gear_1 actions: - flow_call: - flow: "add_item_to_character" + flow: add_item_to_character inputs: character: "outputs.knave" item: "{{ result }}" - - table: "general_gear_2" + - table: general_gear_2 actions: - flow_call: - flow: "add_item_to_character" + flow: add_item_to_character inputs: character: "outputs.knave" item: "{{ result }}" additional_actions: - flow_call: - flow: "add_item_to_character" + flow: add_item_to_character inputs: character: "outputs.knave" - item: "rations" + item: rations quantity: 2 - - id: "generate_traits" + - id: generate_traits name: "Determine Character Traits" - type: "table_roll" + type: table_roll prompt: "Rolling for your character's physical and personality traits..." parallel: true tables: - - table: "physique" + - table: physique actions: - set_value: path: "outputs.knave.traits.physique" value: "{{ result }}" - - table: "face" + - table: face actions: - set_value: path: "outputs.knave.traits.face" value: "{{ result }}" - - table: "skin" + - table: skin actions: - set_value: path: "outputs.knave.traits.skin" value: "{{ result }}" - - table: "hair" + - table: hair actions: - set_value: path: "outputs.knave.traits.hair" value: "{{ result }}" - - table: "clothing" + - table: clothing actions: - set_value: path: "outputs.knave.traits.clothing" value: "{{ result }}" - - table: "virtue" + - table: virtue actions: - set_value: path: "outputs.knave.traits.virtue" value: "{{ result }}" - - table: "vice" + - table: vice actions: - set_value: path: "outputs.knave.traits.vice" value: "{{ result }}" - - table: "speech" + - table: speech actions: - set_value: path: "outputs.knave.traits.speech" value: "{{ result }}" - - table: "background" + - table: background actions: - set_value: path: "outputs.knave.traits.background" value: "{{ result }}" - - table: "misfortunes" + - table: misfortunes actions: - set_value: path: "outputs.knave.traits.misfortunes" value: "{{ result }}" - - table: "alignment" + - table: alignment actions: - set_value: path: "outputs.knave.traits.alignment" value: "{{ result }}" - - id: "choose_gender_choice" + - id: choose_gender_choice name: "Gender Determination Choice" - type: "player_choice" + type: player_choice prompt: "How would you like to determine your character's gender?" choices: - - id: "player_choice" + - id: player_choice label: "Player Choice" next_step: "choose_gender" - - id: "random_roll" + - id: random_roll label: "Randomly Choose" next_step: "roll_gender" - - id: "choose_gender" + - id: choose_gender name: "Choose Character Gender" - type: "player_choice" + type: player_choice prompt: "Select your gender" choices: - - id: "female" + - id: female label: "Female" actions: - set_value: path: "outputs.knave.gender" value: "Female" - - id: "male" + - id: male label: "Male" actions: - set_value: path: "outputs.knave.gender" value: "Male" - - id: "non_binary" + - id: non_binary label: "Non-binary" actions: - set_value: path: "outputs.knave.gender" value: "Non-binary" - next_step: "choose_name" + next_step: choose_name - - id: "roll_gender" + - id: roll_gender name: "Determine Character Gender" - type: "table_roll" + type: table_roll prompt: "Rolling for your character's gender..." tables: - table: gender @@ -268,53 +270,53 @@ steps: - set_value: path: "outputs.knave.gender" value: "{{ result }}" - next_step: "choose_name" + next_step: choose_name - - id: "choose_name" + - id: choose_name name: "Choose Character Name" - type: "player_input" + type: player_input prompt: "What is your character's name?" actions: - set_value: path: "outputs.knave.name" value: "{{ result }}" - - id: "character_description" + - id: character_description name: "Generate Character Description" - type: "llm_generation" + type: llm_generation condition: "llm_enabled" - prompt_id: "character_description" + prompt_id: character_description prompt_data: name: "{{ outputs.knave.name }}" gender: "{{ outputs.knave.gender }}" traits: "{{ outputs.knave.traits }}" llm_settings: - provider: "ollama" - model: "gemma3" + provider: ollama + model: gemma3 max_tokens: 200 actions: - set_value: path: "outputs.knave.description" value: "{{ result }}" - - id: "review_character" + - id: review_character name: "Review and Confirm Character" - type: "player_choice" + type: player_choice prompt: "Here is your completed character:" pre_actions: - display_value: "outputs.knave" choices: - - id: "accept" + - id: accept label: "Accept this character" next_step: "finalize_character" - - id: "reroll" + - id: reroll label: "Start over with a new character" next_step: "roll_abilities" reset_outputs: true - - id: "finalize_character" + - id: finalize_character name: "Character Creation Complete" - type: "completion" + type: completion prompt: "Your Knave is ready for adventure!" actions: - validate_value: "outputs.knave" diff --git a/systems/knave_1e/flows/perform_saving_throw.yaml b/systems/knave_1e/flows/perform_saving_throw.yaml new file mode 100644 index 0000000..cf8a980 --- /dev/null +++ b/systems/knave_1e/flows/perform_saving_throw.yaml @@ -0,0 +1,113 @@ +id: perform_saving_throw +type: flow +name: "Perform Saving Throw" +description: >- + Determines the saving throw required from a given context and performs it. +version: "1.0" +inputs: + - type: character + id: actor + description: "The character for whom the saving throw is being performed." + - type: str + id: context_summary + description: "A summary of the context in which the saving throw should be being made." + +outputs: + - type: str + id: saving_throw_result + description: "The result of the saving throw determination." + +variables: + - type: str + id: saving_throw_ability + description: "The ability score used for the saving throw." + - type: str + id: saving_throw_type + description: "The type of saving throw being made (basic, advantage, disadvantage)." + +steps: + - id: determine_saving_throw_ability + name: "Determine Saving Throw Ability" + type: llm_generation + prompt_id: determine_saving_throw_ability + prompt_data: + context_summary: "{{ inputs.context_summary }}" + validation: + type: "json_schema" + schema: + type: "object" + properties: + ability: + type: "string" + enum: + [ + "strength", + "dexterity", + "constitution", + "intelligence", + "wisdom", + "charisma", + ] + reason: + type: "string" + minLength: 10 + required: ["ability", "reason"] + actions: + - set_value: + path: "variables.saving_throw_ability" + value: "{{ llm_result.ability }}" + - log_message: + message: "Determined Ability: {{ variables.saving_throw_ability }}" + - log_message: + message: "Justification: {{ llm_result.reason }}" + + - id: determine_saving_throw_type + name: "Determine Saving Throw Type" + type: llm_generation + prompt_id: determine_saving_throw_type + prompt_data: + context_summary: "{{ inputs.context_summary }}" + saving_throw_ability: "{{ variables.saving_throw_ability }}" + validation: + type: "json_schema" + schema: + type: "object" + properties: + type: + type: "string" + enum: ["basic", "advantage", "disadvantage"] + reason: + type: "string" + minLength: 10 + required: ["type", "reason"] + actions: + - set_value: + path: "variables.saving_throw_type" + value: "{{ llm_result.type }}" + - log_message: + message: "Determined Roll Type: {{ llm_result.type | capitalize }}" + - log_message: + message: "Justification: {{ llm_result.reason }}" + + - id: perform_saving_throw + name: "Perform Saving Throw" + type: flow_call + flow: roll_saving_throw + inputs: + saving_throw_type: "{{ variables.saving_throw_type }}" + saving_throw_modifier: "{{ inputs.actor.abilities[variables.saving_throw_ability].bonus }}" + saving_throw_dc: 15 + actions: + # for this step, result is the "outputs" of the sub-flow + - set_value: + path: "outputs.saving_throw_result" + value: "{{ result.saving_throw_result }}" + + - id: Evaluate_saving_throw_result + name: "Evaluate Saving Throw Result" + type: "completion" + result_message: "Saving throw complete - {% if outputs.saving_throw_result %}Success{% else %}Failure{% endif %}" + # actions: + # - set_value: + # path: "outputs.saving_throw_result" + # value: "{{ outputs.saving_throw_result }}" diff --git a/systems/knave_1e/flows/roll_saving_throw.yaml b/systems/knave_1e/flows/roll_saving_throw.yaml index 094e193..6a4e82e 100644 --- a/systems/knave_1e/flows/roll_saving_throw.yaml +++ b/systems/knave_1e/flows/roll_saving_throw.yaml @@ -20,14 +20,17 @@ outputs: description: "Whether the saving throw was successful" variables: - base_dice_roll: "" - roll_result: "" + - type: str + id: base_dice_roll + description: "The base dice roll to use for the saving throw" + - type: roll_result + id: saving_throw_result + description: "The result of the saving throw roll" steps: - id: "determine_dice_roll" name: "Determine Saving Throw Type" type: "conditional" - result_message: "Selected {{ inputs.saving_throw_type }} dice ({{ variables.base_dice_roll }})" if: "{{ inputs.saving_throw_type == 'basic' }}" then: # Basic @@ -46,19 +49,18 @@ steps: - set_value: path: "variables.base_dice_roll" value: 2d20kl1 - next_step: "execute_saving_throw_roll" + result_message: "Selected {{ inputs.saving_throw_type }} dice ({{ variables.base_dice_roll }})" - - id: "execute_saving_throw_roll" - name: "Roll Saving Throw" - type: "dice_roll" - result_message: "Rolled {{ variables.roll_result }} vs DC {{ inputs.saving_throw_dc }}" - prompt: "Rolling {{ variables.base_dice_roll }} + {{ inputs.saving_throw_modifier }} for saving throw..." + - id: execute_saving_throw_roll + type: dice_roll roll: "{{ variables.base_dice_roll }} + {{ inputs.saving_throw_modifier }}" actions: - set_value: - path: "variables.roll_result" + path: "variables.saving_throw_result" value: "{{ result }}" - next_step: "evaluate_saving_throw_result" + - log_message: + message: "Rolled {{ result.description }} for saving throw" + result_message: "Rolled {{ result.total }} vs DC {{ inputs.saving_throw_dc }}" - id: evaluate_saving_throw_result name: "Evaluate Saving Throw Result" @@ -67,4 +69,4 @@ steps: actions: - set_value: path: "outputs.saving_throw_result" - value: "{{ variables.roll_result > inputs.saving_throw_dc }}" + value: "{{ variables.saving_throw_result.total > inputs.saving_throw_dc }}" diff --git a/systems/knave_1e/prompts/character-description.yaml b/systems/knave_1e/prompts/character-description.yaml index e263fac..1abf307 100644 --- a/systems/knave_1e/prompts/character-description.yaml +++ b/systems/knave_1e/prompts/character-description.yaml @@ -1,12 +1,12 @@ -id: "character_description" -kind: "prompt" +id: character_description +kind: prompt name: "Character Description Generator" description: "Generates descriptive text for newly created Knave characters" version: "1.0" llm: - provider: "ollama" - model: "gemma3" + provider: ollama + model: gemma2 prompt_template: | Please generate me a paragraph description (~125 words) for a character for the tabletop role-playing game Knave (1st edition) who has the following traits: diff --git a/systems/knave_1e/prompts/determine-saving-throw-ability.yaml b/systems/knave_1e/prompts/determine-saving-throw-ability.yaml new file mode 100644 index 0000000..c7be6c0 --- /dev/null +++ b/systems/knave_1e/prompts/determine-saving-throw-ability.yaml @@ -0,0 +1,81 @@ +id: determine_saving_throw_ability +kind: prompt +name: "Determine Saving Throw Ability" +description: "Determines the appropriate ability score for a saving throw based on the context of the situation." +version: "1.0" + +llm: + provider: ollama + model: gemma2 + +prompt_template: | + We are playing Knave 1st edition. Please help me determine which Ability should be used for an Saving Throw based on the following context: + + CONTEXT: + "{{ context_summary }}" + + The possible Abilities are: + - strength + - dexterity + - constitution + - intelligence + - wisdom + - charisma + + The rules for what abilities apply in what situations in Knave are as follows: + + Each of the six abilities is used in different circumstances. + * Strength: Used for melee attacks and saves requiring + physical power, like lifting gates, bending bars, etc. + * Dexterity: Used for saves requiring poise, speed, and + reflexes, like dodging, climbing, sneaking, balancing, etc. + * Constitution: Used for saves to resist poison, sickness, + cold, etc. The Constitution bonus is added to healing + rolls. A PC's number of item slots is always equal to their + Constitution defense. + * Intelligence: Used for saves requiring concentration + and precision, such as wielding magic, resisting magical + effects, recalling lore, crafting objects, tinkering with + machinery, picking pockets, etc. + * Wisdom: Used for ranged attacks and saves requiring + perception and intuition, such as tracking, navigating, + searching for secret doors, detecting illusions, etc. + * Charisma: Used for saves to persuade, deceive, interro- + gate, intimidate, charm, provoke, etc. + + Please provide your response in the exact JSON format below, with no other text: + { + "ability": , + "reason": + } + + Here are some example inputs and good responses: + + INPUT 1: + Eira (half-elf) is at the entrance to an ancient ruin. She is attempting to pick an ancient lock on sturdy door using Thieves' Tools. The lock appears well-maintained, rusty hinges may make noise if disturbed. Eira is attempting to bypass the lock without triggering any traps or alarms. + + GOOD RESPONSE 1: + { + "ability": "intelligence", + "reason": "Intelligence should be used for actions that require precision and for tinkering with machinery." + } + + INPUT 2: + The party is in a dimly lit tavern, attempting to gather information about a recent string of burglaries in the nearby town. One player, attempting to strike up a conversation with a gruff-looking local blacksmith who is repairing a broken sword, leans against the workbench and asks him if he's heard anything unusual about the thefts. The blacksmith, suspicious of outsiders, glances around the room and seems wary of speaking further. A hooded figure at the end of the bar catches his eye, and the blacksmith subtly shifts his attention away from the group. + + GOOD RESPONSE 2: + { + "ability": "charisma", + "reason": "The party is trying to persuade and gather information from the blacksmith, making Charisma the most relevant ability." + } + + INPUT 3: + + The player is trapped within a crumbling, subterranean temple dedicated to a forgotten deity. As he cautiously proceeds through a narrow corridor, a sudden, violent tremor shakes the temple. Dust and debris rain down from the ceiling. They should roll a save for whether they are able to avoid the falling debris. + + GOOD RESPONSE 3: + + { + "ability": "dexterity", + "reason": "Dexterity is used for saves requiring poise, speed, and reflexes, which would be necessary to avoid falling debris." + } diff --git a/systems/knave_1e/prompts/determine-saving-throw-type.yaml b/systems/knave_1e/prompts/determine-saving-throw-type.yaml new file mode 100644 index 0000000..281d3c0 --- /dev/null +++ b/systems/knave_1e/prompts/determine-saving-throw-type.yaml @@ -0,0 +1,87 @@ +id: determine_saving_throw_type +kind: prompt +name: "Determine Saving Throw Type" +description: >- + Determines the type (basic, advantage, disadvantage) of + saving throw being made based on the context of the + situation. +version: "1.0" +llm: + provider: "ollama" + model: "gemma2" + +prompt_template: | + We are playing Knave 1st edition. Please help me determine which + Type should be used for an Saving Throw (basic, advantage, disadvantage) + based on the following context: + + "{{ context_summary }} The ability being tested is {{ saving_throw_ability }}." + + Be sure to consider any situational factors that may make the saving + throw easier or harder. When in doubt, err on the side of a basic roll. + + The possible Types are: + - basic + - advantage + - disadvantage + + The rules for when advantage or disadvantage are applied in Knave + are as follows: + + "If there are situational factors that make a save significantly + easier or harder, the referee may grant the roll advantage or + disadvantage. If a roll has advantage, roll 2d20 and use the + better of the two dice. If it has disadvantage, roll 2d20 and + use the worse of the two dice." + + Please provide your response in the exact JSON format below, + with no other text: + { + "type": , + "reason": + } + + Here are some example inputs and good responses: + + INPUT 1: + Eira (half-elf) is at the entrance to an ancient ruin. She is + attempting to pick an ancient lock on sturdy door using Thieves' + Tools. The lock appears well-maintained, rusty hinges may make + noise if disturbed. Eira is attempting to bypass the lock without + triggering any traps or alarms. She has recently discovered a tome + detailing ancient lock mechanisms which she is using. The ability + being tested is intelligence. + + GOOD RESPONSE 1: + { + "type": "advantage", + "reason": "The tome on ancient lock mechanisms provides Eira with valuable insights." + } + + INPUT 2: + The party is in a dimly lit tavern, attempting to gather information + about a recent string of burglaries in the nearby town. One player, + attempting to strike up a conversation with a gruff-looking local + blacksmith who is repairing a broken sword, leans against the + workbench and asks him if he's heard anything unusual about the + thefts. The blacksmith, suspicious of outsiders, glances around the + room and seems wary of speaking further. A hooded figure at the end + of the bar catches his eye, and the blacksmith subtly shifts his + attention away from the group. The ability being tested is charisma. + + GOOD RESPONSE 2: + { + "type": "disadvantage", + "reason": "The blacksmith is suspicious of outsiders and is aware of a hooded figure watching." + } + + INPUT 3: + The player is battling a wyrm while standing on the edge of a rocky + cliffside. The wyrm has used its wings to blow a gust of wind at the + player causing him to possibly lose his footing and fall from the cliff. + + GOOD RESPONSE 3: + { + "type": "basic", + "reason": "Although the wyrm's gust of wind is a significant threat, it is the cause for the check. There are no additional conditions that would grant advantage or disadvantage." + }