Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [0.0.4]

### Added

- `CustomAnswersStrategy`: new strategy that returns preconfigured answers for exact-match questions.

## [0.0.3]

### Added
Expand Down
33 changes: 32 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ OpenAI-compatible mock server for testing LLM integrations.
- Default mirror strategy (echoes input as output)
- **Tool calling support** — trigger phrase–driven tool call responses when `tools` are present in the request using `call tool '<name>' with '<json>'`
- **Error simulation** — trigger phrase–driven error responses using `raise error <json>` in the last user message
- **Custom answers** — return preconfigured answers for exact-match questions via `customReplies` in `config.yaml`
- Streaming support for both Chat Completions and Responses APIs (including `stream_options.include_usage`)

## Quick Start
Expand Down Expand Up @@ -99,12 +100,20 @@ cors:
- "http://localhost:8000"

# Ordered list of strategies to try (first non-empty result wins)
# Available: ErrorStrategy, ToolCallStrategy, MirrorStrategy
# Available: ErrorStrategy, CustomAnswersStrategy, ToolCallStrategy, MirrorStrategy
strategies:
- ErrorStrategy
- CustomAnswersStrategy
- ToolCallStrategy
- MirrorStrategy

# Inline question→answer pairs for CustomAnswersStrategy (exact-match, case-sensitive)
customReplies:
- question: How are you today?
answer: I'm fine. how are you?
- question: What is 1+1?
answer: 2

models:
- id: "gpt-4o"
created: 1715367049
Expand Down Expand Up @@ -218,6 +227,28 @@ client.chat.completions.create(
Only the last user message is checked. System/assistant/tool messages are ignored.
Works on both `/chat/completions` and `/responses` endpoints.

### Custom Answers

When `CustomAnswersStrategy` is included in the `strategies` list, llmock checks the last user message against a list of preconfigured question/answer pairs (exact match, case-sensitive, whitespace trimmed). Configure the pairs inline in `config.yaml`:

```yaml
strategies:
- ErrorStrategy
- CustomAnswersStrategy
- ToolCallStrategy
- MirrorStrategy

customReplies:
- question: How are you today?
answer: I'm fine. how are you?
- question: What is 1+1?
answer: 2
```

If the message matches a `question`, the corresponding `answer` is returned. If no match is found, the strategy falls through to the next one (e.g. `MirrorStrategy`).

Works on both `/chat/completions` and `/responses` endpoints.

## Development

### Run Tests
Expand Down
8 changes: 8 additions & 0 deletions config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,13 @@ models:
# Ordered list of strategies (first non-empty result wins)
strategies:
- ErrorStrategy
- CustomAnswersStrategy
- ToolCallStrategy
- MirrorStrategy

# Inline question→answer pairs for CustomAnswersStrategy (exact-match, case-sensitive)
# customReplies:
# - question: How are you today?
# answer: I'm fine. how are you?
# - question: What is 1+1?
# answer: 2
8 changes: 8 additions & 0 deletions docs/llmock-skill/references/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,13 @@ models:
# Ordered list of strategies (first non-empty result wins)
strategies:
- ErrorStrategy
- CustomAnswersStrategy
- ToolCallStrategy
- MirrorStrategy

# Inline question→answer pairs for CustomAnswersStrategy (exact-match, case-sensitive)
# customReplies:
# - question: How are you today?
# answer: I'm fine. how are you?
# - question: What is 1+1?
# answer: 2
8 changes: 8 additions & 0 deletions src/llmock/strategies/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,21 @@
ChatErrorStrategy,
ResponseErrorStrategy,
)
from llmock.strategies.strategy_custom_answers import (
ChatCustomAnswersStrategy,
ResponseCustomAnswersStrategy,
)
from llmock.strategies.strategy_tool_call import (
ChatToolCallStrategy,
ResponseToolCallStrategy,
)

# Short strategy names used in config → (ChatClass, ResponseClass)
_STRATEGIES: dict[str, tuple[type, type]] = {
"CustomAnswersStrategy": (
ChatCustomAnswersStrategy,
ResponseCustomAnswersStrategy,
),
"ErrorStrategy": (ChatErrorStrategy, ResponseErrorStrategy),
"MirrorStrategy": (ChatMirrorStrategy, ResponseMirrorStrategy),
"ToolCallStrategy": (ChatToolCallStrategy, ResponseToolCallStrategy),
Expand Down
7 changes: 6 additions & 1 deletion src/llmock/strategies/strategy_composition.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@

logger = logging.getLogger(__name__)

_DEFAULT_STRATEGIES = ["ErrorStrategy", "ToolCallStrategy", "MirrorStrategy"]
_DEFAULT_STRATEGIES = [
"ErrorStrategy",
"CustomAnswersStrategy",
"ToolCallStrategy",
"MirrorStrategy",
]


class ChatCompositionStrategy:
Expand Down
81 changes: 81 additions & 0 deletions src/llmock/strategies/strategy_custom_answers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
"""Custom answers strategy - return preconfigured answers for exact-match questions.

Reads the ``customReplies`` list directly from config. The list must have
the following structure in ``config.yaml``::

customReplies:
- question: How are you today?
answer: I'm fine. how are you?
- question: What is 1+1?
answer: 2

When the last user message exactly matches a ``question`` (case-sensitive,
stripped of leading/trailing whitespace), the corresponding ``answer`` is
returned as a text response. If no match is found, an empty list is
returned so the next strategy in the composition chain runs.

If ``customReplies`` is not present in config the strategy passes through
silently (empty list).
"""

import logging
from typing import Any

from llmock.schemas.chat import ChatCompletionRequest
from llmock.schemas.responses import ResponseCreateRequest
from llmock.strategies.base import StrategyResponse, text_response
from llmock.utils.chat import (
extract_last_user_text_chat,
extract_last_user_text_response,
)

logger = logging.getLogger(__name__)

CONFIG_KEY = "customReplies"


def _build_replies(entries: list[Any]) -> dict[str, str]:
"""Build a question→answer mapping from a list of config entries."""
return {
str(entry["question"]).strip(): str(entry["answer"])
for entry in entries
if isinstance(entry, dict) and "question" in entry and "answer" in entry
}


def _match(content: str | None, replies: dict[str, str]) -> list[StrategyResponse]:
"""Return a response if *content* exactly matches a question, else []."""
if content is None:
return []
answer = replies.get(content.strip())
if answer is None:
return []
return [text_response(answer)]


class ChatCustomAnswersStrategy:
"""Exact-match custom answers strategy for the Chat Completions API."""

def __init__(self, config: dict[str, Any]) -> None:
entries = config.get(CONFIG_KEY, [])
self._replies: dict[str, str] = _build_replies(entries)

def generate_response(
self, request: ChatCompletionRequest
) -> list[StrategyResponse]:
content = extract_last_user_text_chat(request)
return _match(content, self._replies)


class ResponseCustomAnswersStrategy:
"""Exact-match custom answers strategy for the Responses API."""

def __init__(self, config: dict[str, Any]) -> None:
entries = config.get(CONFIG_KEY, [])
self._replies: dict[str, str] = _build_replies(entries)

def generate_response(
self, request: ResponseCreateRequest
) -> list[StrategyResponse]:
content = extract_last_user_text_response(request)
return _match(content, self._replies)
Loading
Loading