diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b323142 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +.github +.venv +.env +.env.* +!.env.example +.idea +.pytest_cache +__pycache__ +*.py[cod] +.DS_Store +chroma_db +sync_state.json +docs/assets/*.gif diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3682b7e --- /dev/null +++ b/.env.example @@ -0,0 +1,16 @@ +# Copy this file to .env and fill in real values locally. +# Never commit .env with secrets or private workspace IDs. + +OPENAI_API_KEY=your_openai_api_key +TELEGRAMBOT_API_KEY=your_telegram_bot_token +NOTION_API_KEY=your_notion_integration_secret +NOTION_DATABASE_ID=your_notion_database_id + +# Optional proxy for OpenAI/HTTP clients. +PROXY_URL= + +# Local persistence and collection names. +CHROMA_PATH=./chroma_db +BUSINESS_CASES_COLLECTION=business_cases +MEMORY_COLLECTION=conversation_memory +SYNC_STATE_FILE=sync_state.json diff --git a/.github/workflows/tests.yml b/.github/workflows/ci.yml similarity index 93% rename from .github/workflows/tests.yml rename to .github/workflows/ci.yml index 9d7377a..28838e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Tests +name: CI on: push: @@ -23,7 +23,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install -r requirements-dev.txt - name: Run automated tests run: python -m pytest -q diff --git a/.gitignore b/.gitignore index 83b9869..a763ec8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .venv/ .env +.env.* +!.env.example __pycache__/ .pytest_cache/ *.py[cod] diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e24ce8a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD ["python", "telegram_bot.py"] diff --git a/README.md b/README.md index ee0a4b9..ebeadee 100644 --- a/README.md +++ b/README.md @@ -1,103 +1,256 @@ -# Consulting Assistant +# AI Consulting Assistant -AI-консультант в формате Telegram-бота для бизнес-задач, связанных с внедрением искусственного интеллекта. Бот умеет вести обычный чат, запускать многоагентный сценарий бизнес-консультации, сохранять новые бизнес-кейсы в Notion и искать сохраненные кейсы через локальную RAG-базу знаний на ChromaDB. +**RU summary:** Telegram AI-ассистент для бизнес-консультаций по внедрению AI. Проект объединяет multi-agent workflow на CrewAI, RAG-поиск по кейсам в ChromaDB, синхронизацию кейсов из Notion, память диалогов и обработку голосовых сообщений через Whisper. -## Что Умеет Проект +## Overview -- Предоставляет Telegram-интерфейс с режимами чата, бизнес-консультации, помощи и сохранения кейсов. -- Использует CrewAI-сценарий с агентами исследователя, консультанта и критика для вопросов по бизнес-консалтингу. -- Ищет AI-бизнес-кейсы, сохраненные в ChromaDB, через `rag_tool.py`. -- Хранит память диалогов в ChromaDB через `memory.py`. -- Сохраняет структурированные бизнес-кейсы в Notion через `scribe.py`. -- Синхронизирует бизнес-кейсы из Notion в ChromaDB через Notion-to-Chroma скрипты. +AI Consulting Assistant is a Telegram-based portfolio project that demonstrates how an LLM application can support business consulting workflows around AI adoption. The bot can answer regular chat questions, run a structured multi-agent consultation, search a local business-case knowledge base with RAG, save new cases to Notion, and keep lightweight conversation memory. -## Основные Файлы +The project is intentionally MVP-sized, but it includes production-oriented building blocks: environment-based configuration, Docker support, CI, automated tests with mocks, local persistent ChromaDB storage, and clear documentation. -- `telegram_bot.py` - точка входа Telegram-бота, меню, команды и обработка сообщений. -- `orchestrator.py` - маршрутизация сообщений между обычным чатом и режимом бизнес-консультации. -- `agents.py` - настройка агентов и вспомогательная логика. -- `rag_tool.py` - инструмент поиска бизнес-кейсов в ChromaDB. -- `memory.py` - сохранение и поиск памяти диалогов. -- `scribe.py` - создание новых страниц с бизнес-кейсами в Notion. -- `notion_to_chromadb.py` - полная пересборка базы ChromaDB из Notion. -- `sync_notion_to_chromadb.py` - инкрементальная синхронизация Notion -> ChromaDB. -- `digest.py` - логика генерации дайджеста. -- `main.py` - минимальный стартовый файл с загрузкой переменных окружения. -- `requirements.txt` - зависимости Python. +## Problem -## Переменные Окружения +Small teams exploring AI adoption often ask broad questions such as "How can we automate support?" or "Are there real examples of AI agents in sales?" A useful assistant should not only generate generic recommendations; it should ground answers in reusable cases, expose risks, and let the team grow its own knowledge base over time. -Создайте локальный файл `.env` в корне проекта. Не коммитьте его в Git. +This project addresses that workflow with: -Обязательные переменные: +- a Telegram interface for quick access; +- business consultation mode with Researcher, Consultant, and Critic agents; +- RAG search over curated business cases; +- Notion as a lightweight case-management backend; +- local ChromaDB for embeddings and retrieval. -```env -OPENAI_API_KEY=your_openai_key -TELEGRAMBOT_API_KEY=your_telegram_bot_token -NOTION_API_KEY=your_notion_integration_secret -PROXY_URL=optional_proxy_url +## Key Features + +- **Telegram bot UX:** main menu, chat mode, business consultation mode, save-case flow, help/info screens. +- **Multi-agent consultation:** CrewAI Researcher, Consultant, and Critic agents collaborate on business AI questions. +- **RAG knowledge base:** `ChromaRAGTool` retrieves AI business cases from ChromaDB using OpenAI embeddings. +- **Notion case storage:** structured case creation through the Notion API. +- **Notion -> ChromaDB sync:** full rebuild and incremental sync scripts keep RAG data fresh. +- **Conversation memory:** previous user turns are stored and retrieved from ChromaDB. +- **Voice input:** Telegram voice messages are transcribed with Whisper before being routed to chat or consultation mode. +- **Photo analysis in chat mode:** image messages can be sent to the OpenAI vision-capable chat endpoint. +- **Tests and CI:** pytest suite with fakes/mocks, plus GitHub Actions. +- **Dockerized runtime:** Dockerfile and Compose setup with a ChromaDB volume. + +## Architecture + +```mermaid +flowchart LR + U["Telegram User"] --> B["Telegram Bot
telegram_bot.py"] + B --> O["Orchestrator
orchestrator.py"] + O --> C["Chat Mode
direct OpenAI response"] + O --> K["Consultation Mode
CrewAI workflow"] + B --> S["Save Case Mode
scribe.py"] + S --> N["Notion Database"] + O --> M["Conversation Memory
memory.py"] + M --> DB["ChromaDB"] +``` + +```mermaid +flowchart LR + Q["Business Question"] --> R["Researcher Agent"] + R --> T["Business Cases Search
rag_tool.py"] + T --> V["ChromaDB
business_cases"] + V --> R + R --> A["Consultant Agent"] + A --> C["Critic Agent"] + C --> F["Final Telegram Answer"] +``` + +More details are available in [docs/architecture.md](docs/architecture.md). + +## Multi-Agent Workflow + +The consultation mode is designed as a three-step review loop: + +1. **Researcher** searches the RAG knowledge base for relevant business cases, implementation patterns, tools, and outcomes. +2. **Consultant** turns the retrieved context into a practical recommendation with suggested architecture, expected results, and next steps. +3. **Critic** reviews the recommendation for missing evidence, implementation risks, hidden costs, data readiness, and compliance concerns. + +This structure is intentionally more conservative than a single prompt because it separates retrieval, recommendation, and risk review. + +## RAG Pipeline + +Business cases are stored in Notion and synchronized into a local ChromaDB collection: + +```mermaid +flowchart LR + N["Notion DB
business cases"] --> SY["Sync scripts
notion_to_chromadb.py
sync_notion_to_chromadb.py"] + SY --> E["OpenAI Embeddings
text-embedding-3-small"] + E --> C["ChromaDB
business_cases collection"] + C --> R["RAG Search
rag_tool.py"] + R --> A["CrewAI Researcher"] ``` -## Установка +See [docs/rag_pipeline.md](docs/rag_pipeline.md) for implementation notes. + +## Notion Integration + +The bot can save a new business case from Telegram into Notion using `scribe.py`. A case includes title, category, use case, tools, summary, implementation details, pros, cons, source, and date. + +Notion synchronization is handled by: + +- `notion_to_chromadb.py` for a full rebuild of the `business_cases` ChromaDB collection; +- `sync_notion_to_chromadb.py` for incremental sync based on Notion `last_edited_time`. + +The Notion database id is configured through `NOTION_DATABASE_ID` and is not committed to the repository. + +## Voice Input via Whisper + +Telegram voice messages are downloaded as audio files, transcribed with Whisper, and then routed through the selected mode: + +- chat mode sends the transcript to direct chat; +- consultation mode sends it to the multi-agent workflow; +- auto mode chooses based on keywords. + +The voice path is kept inside `telegram_bot.py`, while pure helper logic is tested separately. + +## Tech Stack + +- Python 3.11 +- python-telegram-bot +- OpenAI API: chat, embeddings, Whisper transcription +- CrewAI +- ChromaDB +- Notion API +- pytest and pytest-asyncio +- Docker and Docker Compose +- GitHub Actions CI + +## Demo + +The repository includes mock demo screenshots based on the implemented Telegram bot flows. They are not live Telegram screenshots and do not use real tokens. + +> UI flow mock based on implemented bot flows. + +| Main menu | Consultation mode | +|---|---| +| ![Main menu mock](docs/assets/main_menu.png) | ![Consultation mock](docs/assets/business_consultation.png) | + +| RAG result | Save case | Voice transcription | +|---|---|---| +| ![RAG result mock](docs/assets/rag_result.png) | ![Notion save mock](docs/assets/notion_save_case.png) | ![Voice flow mock](docs/assets/voice_transcription.png) | + +Optional flow GIF: + +![Mock flow GIF](docs/assets/demo_flow.gif) + +Demo prompts are listed in [docs/demo_queries.md](docs/demo_queries.md). + +## Setup + +Clone the repository and create a virtual environment: ```bash +git clone https://github.com/grinegor/consulting-assistant.git +cd consulting-assistant python -m venv .venv source .venv/bin/activate pip install -r requirements.txt +pip install -r requirements-dev.txt ``` -## Запуск Бота +Create local environment variables: + +```bash +cp .env.example .env +``` + +Fill in `.env`: + +```env +OPENAI_API_KEY=your_openai_api_key +TELEGRAMBOT_API_KEY=your_telegram_bot_token +NOTION_API_KEY=your_notion_integration_secret +NOTION_DATABASE_ID=your_notion_database_id +CHROMA_PATH=./chroma_db +BUSINESS_CASES_COLLECTION=business_cases +MEMORY_COLLECTION=conversation_memory +``` + +Run the bot: ```bash python telegram_bot.py ``` -## Синхронизация Кейсов Из Notion В ChromaDB +Sync Notion cases into ChromaDB: -Полная пересборка коллекции `business_cases`: +```bash +python sync_notion_to_chromadb.py +``` + +For a full rebuild: ```bash python notion_to_chromadb.py ``` -Инкрементальная синхронизация обновленных страниц Notion: +## Docker Setup + +Build and run the Telegram bot service: ```bash -python sync_notion_to_chromadb.py +docker compose up --build ``` -## Тестирование +The Compose setup: -В проекте есть автоматические тесты для основной логики бота, синхронизации Notion/ChromaDB, форматирования RAG-ответов, создания Notion payload через Scribe и eval-проверок маршрутизации. Внешние сервисы, включая Telegram, Notion, OpenAI, ChromaDB и CrewAI, замоканы в тестах, поэтому suite запускается без реальных API-вызовов и секретов. +- reads secrets and ids from `.env`; +- mounts `./chroma_db` into the container for local ChromaDB persistence; +- runs `python telegram_bot.py`. -Запустить все тесты: +Stop the service: ```bash -python -m pytest -q +docker compose down ``` -Запустить только легкие eval-проверки маршрутизации: +## Tests + +Run the full test suite: ```bash -python -m pytest -q -m eval +pytest +``` + +Run the compact CI-style command: + +```bash +python -m pytest -q ``` -Запустить только локальные stress/boundary проверки: +Run eval and stress subsets: ```bash +python -m pytest -q -m eval python -m pytest -q -m stress ``` -Проверить компиляцию Python-файлов проекта: +Compile-check project files: ```bash python -m compileall -q telegram_bot.py scribe.py agents.py digest.py main.py memory.py notion_to_chromadb.py orchestrator.py rag_tool.py sync_notion_to_chromadb.py tests ``` -GitHub Actions автоматически запускает тесты и compile-check при push и pull request в ветку `main`. +Tests use mocks/fakes instead of real Telegram, OpenAI, Notion, CrewAI, or ChromaDB calls. + +## Roadmap + +- Add a web dashboard for reviewing synced cases and retrieval quality. +- Add scheduled Notion synchronization. +- Add structured observability for agent runs and retrieval traces. +- Add richer eval datasets for consultation quality and risk coverage. +- Add deployment manifests for a small cloud VM or container platform. + +## Known Limitations -## Заметки +- MVP-grade local deployment, not a production SaaS backend. +- ChromaDB is local by default and should be backed up or replaced for production. +- No enterprise auth, RBAC, tenant isolation, or audit log yet. +- No production monitoring, alerting, or tracing yet. +- Telegram markdown output may require extra escaping for arbitrary model output. +- Notion schema expectations are currently encoded in the sync scripts. -- `.env`, `.venv`, `.idea` и локальные файлы `chroma_db` намеренно игнорируются Git. -- Локальная база ChromaDB является runtime-данными и должна пересоздаваться или синхронизироваться локально. -- API-ключи и токены бота нужно хранить только в `.env` или в переменных окружения deployment-среды. +More detail is documented in [docs/limitations.md](docs/limitations.md). diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..2aac9b3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +services: + bot: + build: . + environment: + OPENAI_API_KEY: ${OPENAI_API_KEY:?OPENAI_API_KEY is required} + TELEGRAMBOT_API_KEY: ${TELEGRAMBOT_API_KEY:?TELEGRAMBOT_API_KEY is required} + NOTION_API_KEY: ${NOTION_API_KEY:?NOTION_API_KEY is required} + NOTION_DATABASE_ID: ${NOTION_DATABASE_ID:?NOTION_DATABASE_ID is required} + PROXY_URL: ${PROXY_URL:-} + CHROMA_PATH: /app/chroma_db + BUSINESS_CASES_COLLECTION: ${BUSINESS_CASES_COLLECTION:-business_cases} + MEMORY_COLLECTION: ${MEMORY_COLLECTION:-conversation_memory} + SYNC_STATE_FILE: ${SYNC_STATE_FILE:-sync_state.json} + volumes: + - ./chroma_db:/app/chroma_db + restart: unless-stopped diff --git a/dockerfile b/dockerfile deleted file mode 100644 index 3bc07ba..0000000 --- a/dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -cat > Dockerfile << 'EOF' -FROM python:3.11-slim - -WORKDIR /app - -RUN apt-get update && apt-get install -y gcc g++ && rm -rf /var/lib/apt/lists/* - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -COPY . . - -CMD ["python", "telegram_bot.py"] -EOF \ No newline at end of file diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..d0e7480 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,59 @@ +# Architecture + +This project is organized around a Telegram interface, an orchestration layer, a multi-agent consultation workflow, and a local RAG knowledge base backed by ChromaDB. + +## Runtime Flow + +```mermaid +flowchart LR + U["Telegram User"] --> B["Telegram Bot
telegram_bot.py"] + B --> O["Orchestrator
orchestrator.py"] + O --> C["Chat mode
direct OpenAI response"] + O --> K["Consultation mode
CrewAI agents"] + B --> S["Save case mode
scribe.py"] + S --> N["Notion Database"] + O --> M["Conversation memory
memory.py"] + M --> DB["ChromaDB"] +``` + +## Consultation Mode + +```mermaid +flowchart LR + Q["User business question"] --> R["Researcher"] + R --> RT["RAG Tool
Business Cases Search"] + RT --> CDB["ChromaDB
business_cases"] + CDB --> RT + RT --> R + R --> A["Consultant"] + A --> C["Critic"] + C --> F["Final answer
Telegram Markdown"] +``` + +The Researcher is responsible for grounding recommendations in retrieved cases. The Consultant translates context into practical implementation advice. The Critic highlights weak evidence, hidden costs, compliance concerns, and implementation risks. + +## Data Pipeline + +```mermaid +flowchart LR + N["Notion DB
curated AI business cases"] --> SY["Sync scripts"] + SY --> P["Payload builder
documents + metadata + ids"] + P --> E["OpenAI embeddings
text-embedding-3-small"] + E --> C["ChromaDB collection
business_cases"] + C --> R["RAG search"] + R --> AG["Researcher agent"] +``` + +## Key Modules + +- `telegram_bot.py` handles Telegram commands, menus, text messages, voice messages, image messages, and save-case interactions. +- `orchestrator.py` routes messages between chat mode, consultation mode, and auto mode. +- `rag_tool.py` wraps ChromaDB search as a CrewAI tool. +- `memory.py` stores and retrieves conversation memory from ChromaDB. +- `scribe.py` creates structured Notion pages for new business cases. +- `notion_to_chromadb.py` performs a full rebuild from Notion into ChromaDB. +- `sync_notion_to_chromadb.py` performs incremental Notion synchronization. + +## Configuration Boundary + +Secrets and private IDs are loaded from environment variables. The repository includes `.env.example` but does not commit real `.env` values, Notion database IDs, tokens, or ChromaDB data. diff --git a/docs/assets/business_consultation.png b/docs/assets/business_consultation.png new file mode 100644 index 0000000..15d60c8 Binary files /dev/null and b/docs/assets/business_consultation.png differ diff --git a/docs/assets/demo_flow.gif b/docs/assets/demo_flow.gif new file mode 100644 index 0000000..460897c Binary files /dev/null and b/docs/assets/demo_flow.gif differ diff --git a/docs/assets/main_menu.png b/docs/assets/main_menu.png new file mode 100644 index 0000000..9c4ea5e Binary files /dev/null and b/docs/assets/main_menu.png differ diff --git a/docs/assets/notion_save_case.png b/docs/assets/notion_save_case.png new file mode 100644 index 0000000..9204005 Binary files /dev/null and b/docs/assets/notion_save_case.png differ diff --git a/docs/assets/rag_result.png b/docs/assets/rag_result.png new file mode 100644 index 0000000..8d8c3d2 Binary files /dev/null and b/docs/assets/rag_result.png differ diff --git a/docs/assets/voice_transcription.png b/docs/assets/voice_transcription.png new file mode 100644 index 0000000..00ec1cd Binary files /dev/null and b/docs/assets/voice_transcription.png differ diff --git a/docs/demo_queries.md b/docs/demo_queries.md new file mode 100644 index 0000000..d3e0f9c --- /dev/null +++ b/docs/demo_queries.md @@ -0,0 +1,17 @@ +# Demo Queries + +Use these prompts in Telegram business consultation mode to demonstrate the assistant's intended behavior. + +1. "We run an online school. How can we use AI to reduce support workload without hurting response quality?" +2. "Find relevant AI automation cases for document processing in a consulting company." +3. "What would be a practical AI agent pilot for a small sales team using CRM data?" +4. "Compare risks and expected ROI for adding an AI chatbot to a B2B service company." +5. "Suggest an AI workflow for summarizing client calls and creating follow-up tasks." +6. "Which AI use cases are realistic for a marketing agency with a limited budget?" +7. "Give me examples of successful AI implementation in customer support and critique the risks." + +Expected consultation flow: + +1. The Researcher searches the RAG knowledge base. +2. The Consultant turns retrieved cases into an implementation recommendation. +3. The Critic reviews evidence quality, data readiness, hidden costs, and risks. diff --git a/docs/limitations.md b/docs/limitations.md new file mode 100644 index 0000000..54edb0e --- /dev/null +++ b/docs/limitations.md @@ -0,0 +1,33 @@ +# Known Limitations + +This repository is an MVP portfolio project, not a production consulting platform. + +## Runtime + +- The bot uses Telegram long polling, not a horizontally scalable webhook deployment. +- Local ChromaDB is used by default. Production usage would need backups, monitoring, and a migration plan for larger datasets. +- There is no enterprise auth, RBAC, tenant isolation, or audit trail. +- There is no production observability layer for agent traces, retrieval quality, latency, or API costs. + +## Data And Privacy + +- Notion is treated as the source of truth, but schema validation is lightweight. +- ChromaDB may contain private business-case text, so `chroma_db/` is intentionally excluded from Git. +- `.env` must never be committed because it contains API keys and private workspace IDs. + +## Model Behavior + +- The multi-agent flow reduces but does not eliminate hallucination risk. +- The Critic agent can highlight uncertainty, but it is not a formal compliance or security review. +- Telegram Markdown output may need stricter escaping for arbitrary model-generated content. + +## Integrations + +- The Notion sync scripts assume expected property names and simple field types. +- Voice transcription depends on Whisper availability through the OpenAI API. +- Photo analysis is available only in chat mode and depends on model support. + +## Testing Scope + +- Tests use mocks/fakes and do not call real Telegram, OpenAI, Notion, CrewAI, or ChromaDB services. +- Integration tests with real sandbox accounts would be the next step before production deployment. diff --git a/docs/rag_pipeline.md b/docs/rag_pipeline.md new file mode 100644 index 0000000..8464b2f --- /dev/null +++ b/docs/rag_pipeline.md @@ -0,0 +1,76 @@ +# RAG Pipeline + +The RAG pipeline turns curated Notion business cases into searchable context for the multi-agent consultation workflow. + +## Source Of Truth + +Notion is treated as the editable source of truth for business cases. Each page is expected to contain fields such as: + +- Name or Название +- Category +- Use Case +- Summary +- Implementation +- Pros +- Cons +- Tools +- Source +- Date + +## Full Rebuild + +Use the full rebuild script when the local ChromaDB collection should be recreated from scratch: + +```bash +python notion_to_chromadb.py +``` + +The script: + +1. queries the configured Notion database; +2. maps Notion properties into normalized case dictionaries; +3. builds document text, metadata, and ids; +4. recreates the `business_cases` ChromaDB collection; +5. stores documents with OpenAI embeddings. + +## Incremental Sync + +Use incremental sync during normal development: + +```bash +python sync_notion_to_chromadb.py +``` + +The script: + +1. loads `sync_state.json`; +2. fetches all Notion pages; +3. selects pages edited after the last sync timestamp; +4. upserts updated documents into ChromaDB; +5. writes the new sync timestamp. + +## Retrieval + +`rag_tool.py` loads the `business_cases` collection and queries it with the user's business question. Results are formatted with title, category, truncated content, and an approximate relevance score before being passed back to the Researcher agent. + +## Local Persistence + +ChromaDB persists to `CHROMA_PATH`, which defaults to `./chroma_db`. The directory is intentionally ignored by Git because it may contain private data and generated vector index files. + +## Configuration + +Required environment variables: + +```env +OPENAI_API_KEY=your_openai_api_key +NOTION_API_KEY=your_notion_integration_secret +NOTION_DATABASE_ID=your_notion_database_id +``` + +Optional: + +```env +CHROMA_PATH=./chroma_db +BUSINESS_CASES_COLLECTION=business_cases +SYNC_STATE_FILE=sync_state.json +``` diff --git a/memory.py b/memory.py index 8415ea1..2ca54e1 100644 --- a/memory.py +++ b/memory.py @@ -1,12 +1,16 @@ import chromadb from chromadb.utils import embedding_functions import os +import time from dotenv import load_dotenv load_dotenv() +CHROMA_PATH = os.getenv("CHROMA_PATH", "./chroma_db") +MEMORY_COLLECTION = os.getenv("MEMORY_COLLECTION", "conversation_memory") + # Подключаемся к той же ChromaDB -client = chromadb.PersistentClient(path="./chroma_db") +client = chromadb.PersistentClient(path=CHROMA_PATH) # Используем те же эмбеддинги OpenAI openai_ef = embedding_functions.OpenAIEmbeddingFunction( @@ -16,15 +20,13 @@ # Коллекция для памяти диалогов memory_collection = client.get_or_create_collection( - name="conversation_memory", + name=MEMORY_COLLECTION, embedding_function=openai_ef ) def add_to_memory(user_id: str, user_message: str, assistant_response: str): """Сохраняет один оборот диалога в память""" doc = f"Пользователь: {user_message}\nАссистент: {assistant_response}" - # Используем уникальный ID (можно timestamp + user_id) - import time doc_id = f"{user_id}_{int(time.time()*1000)}" memory_collection.upsert( ids=[doc_id], @@ -41,4 +43,4 @@ def retrieve_memory(user_id: str, query: str, n_results=3): ) if results['documents'] and results['documents'][0]: return "\n\n---\n\n".join(results['documents'][0]) - return "" \ No newline at end of file + return "" diff --git a/notion_to_chromadb.py b/notion_to_chromadb.py index 289451a..d78599a 100644 --- a/notion_to_chromadb.py +++ b/notion_to_chromadb.py @@ -9,11 +9,9 @@ load_dotenv() -NOTION_API_KEY = os.getenv("NOTION_API_KEY") -DATABASE_ID = "3538cd80a7f480aab786c93e0c370bf5" PROXY_URL = os.getenv("PROXY_URL") -CHROMA_PATH = "./chroma_db" -COLLECTION_NAME = "business_cases" +CHROMA_PATH = os.getenv("CHROMA_PATH", "./chroma_db") +COLLECTION_NAME = os.getenv("BUSINESS_CASES_COLLECTION", "business_cases") class ProxyOpenAIEmbeddingFunction(embedding_functions.EmbeddingFunction): @@ -39,6 +37,13 @@ def create_openai_client(): ) +def get_notion_database_id(): + database_id = os.getenv("NOTION_DATABASE_ID") + if not database_id: + raise ValueError("NOTION_DATABASE_ID is required") + return database_id + + def safe_get_text(prop, default=""): if not prop: return default @@ -63,7 +68,7 @@ def safe_get_multi_select(prop): return [] ms = prop.get("multi_select", []) if isinstance(ms, list): - return [item.get("name", "") for item in ms if isinstance(item, dict)] + return [item.get("name", "") for item in ms if isinstance(item, dict) and item.get("name")] return [] @@ -83,9 +88,9 @@ def safe_get_date(prop, default=""): def fetch_notion_cases(): - url = f"https://api.notion.com/v1/databases/{DATABASE_ID}/query" + url = f"https://api.notion.com/v1/databases/{get_notion_database_id()}/query" headers = { - "Authorization": f"Bearer {NOTION_API_KEY}", + "Authorization": f"Bearer {os.getenv('NOTION_API_KEY')}", "Content-Type": "application/json", "Notion-Version": "2022-06-28" } diff --git a/orchestrator.py b/orchestrator.py index 201d6c6..9b1abdd 100644 --- a/orchestrator.py +++ b/orchestrator.py @@ -4,7 +4,6 @@ from crewai import Agent, Task, Crew from rag_tool import ChromaRAGTool from memory import add_to_memory, retrieve_memory -from scribe import ScribeAgent from dotenv import load_dotenv load_dotenv() @@ -95,8 +94,6 @@ verbose=False ) -scribe = ScribeAgent() - def run_business_crew(query: str) -> str: task_research = Task( description=f""" @@ -203,4 +200,4 @@ def orchestrate(user_id: str, user_message: str, mode: str = "auto") -> str: response = direct_chat(user_message, memory_context) add_to_memory(user_id, user_message, response) - return response \ No newline at end of file + return response diff --git a/rag_tool.py b/rag_tool.py index f850eb8..366ba97 100644 --- a/rag_tool.py +++ b/rag_tool.py @@ -6,6 +6,9 @@ load_dotenv() +CHROMA_PATH = os.getenv("CHROMA_PATH", "./chroma_db") +BUSINESS_CASES_COLLECTION = os.getenv("BUSINESS_CASES_COLLECTION", "business_cases") + class ChromaRAGTool(BaseTool): name: str = "Business Cases Search" @@ -24,13 +27,13 @@ def __init__(self, **kwargs): def _ensure_initialized(self): if self._client is None: - object.__setattr__(self, '_client', chromadb.PersistentClient(path="./chroma_db")) + object.__setattr__(self, '_client', chromadb.PersistentClient(path=CHROMA_PATH)) object.__setattr__(self, '_openai_ef', embedding_functions.OpenAIEmbeddingFunction( api_key=os.getenv("OPENAI_API_KEY"), model_name="text-embedding-3-small" )) object.__setattr__(self, '_collection', self._client.get_collection( - name="business_cases", + name=BUSINESS_CASES_COLLECTION, embedding_function=self._openai_ef )) @@ -71,4 +74,4 @@ def _run(self, query: str) -> str: if __name__ == "__main__": tool = ChromaRAGTool() result = tool._run("автоматизация поддержки клиентов") - print(result) \ No newline at end of file + print(result) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..83b2c4b --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +-r requirements.txt + +pytest +pytest-asyncio diff --git a/requirements.txt b/requirements.txt index fb1f424..cfb3534 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,3 @@ feedparser openai httpx python-dotenv - -# Test/development dependencies -pytest -pytest-asyncio diff --git a/scribe.py b/scribe.py index ed512e4..d689e8d 100644 --- a/scribe.py +++ b/scribe.py @@ -5,6 +5,21 @@ load_dotenv() +def build_notion_case_properties(case_data: dict) -> dict: + tools_list = [t.strip() for t in case_data.get("tools", "").split(",") if t.strip()] + return { + "Name": {"title": [{"text": {"content": case_data.get("title", "")}}]}, + "Category": {"select": {"name": case_data.get("category", "")}}, + "Use Case": {"rich_text": [{"text": {"content": case_data.get("use_case", "")}}]}, + "Summary": {"rich_text": [{"text": {"content": case_data.get("summary", "")}}]}, + "Implementation": {"rich_text": [{"text": {"content": case_data.get("implementation", "")}}]}, + "Pros": {"rich_text": [{"text": {"content": case_data.get("pros", "")}}]}, + "Cons": {"rich_text": [{"text": {"content": case_data.get("cons", "")}}]}, + "Source": {"url": case_data.get("source", "")}, + "Date": {"date": {"start": case_data.get("date", "")}}, + "Tools": {"multi_select": [{"name": tool} for tool in tools_list]} + } + class NotionCreateCaseTool(BaseTool): name: str = "Create Business Case in Notion" @@ -16,28 +31,17 @@ class NotionCreateCaseTool(BaseTool): def __init__(self, **kwargs): super().__init__(**kwargs) - # Используем object.__setattr__ для обхода Pydantic + database_id = os.getenv("NOTION_DATABASE_ID") + if not database_id: + raise ValueError("NOTION_DATABASE_ID is required") object.__setattr__(self, '_notion', Client(auth=os.getenv("NOTION_API_KEY"))) - object.__setattr__(self, '_database_id', "3538cd80a7f480aab786c93e0c370bf5") + object.__setattr__(self, '_database_id', database_id) def _run(self, **kwargs) -> str: try: - tools_list = [t.strip() for t in kwargs.get("tools", "").split(",") if t.strip()] - properties = { - "Name": {"title": [{"text": {"content": kwargs.get("title", "")}}]}, - "Category": {"select": {"name": kwargs.get("category", "")}}, - "Use Case": {"rich_text": [{"text": {"content": kwargs.get("use_case", "")}}]}, - "Summary": {"rich_text": [{"text": {"content": kwargs.get("summary", "")}}]}, - "Implementation": {"rich_text": [{"text": {"content": kwargs.get("implementation", "")}}]}, - "Pros": {"rich_text": [{"text": {"content": kwargs.get("pros", "")}}]}, - "Cons": {"rich_text": [{"text": {"content": kwargs.get("cons", "")}}]}, - "Source": {"url": kwargs.get("source", "")}, - "Date": {"date": {"start": kwargs.get("date", "")}}, - "Tools": {"multi_select": [{"name": tool} for tool in tools_list]} - } page = self._notion.pages.create( parent={"database_id": self._database_id}, - properties=properties + properties=build_notion_case_properties(kwargs) ) return f"✅ Кейс «{kwargs.get('title')}» успешно создан в Notion. Ссылка: {page['url']}" except Exception as e: @@ -49,4 +53,4 @@ def __init__(self): self.tool = NotionCreateCaseTool() def create_case(self, case_data: dict) -> str: - return self.tool._run(**case_data) \ No newline at end of file + return self.tool._run(**case_data) diff --git a/sync_notion_to_chromadb.py b/sync_notion_to_chromadb.py index d5e1194..7e022fa 100644 --- a/sync_notion_to_chromadb.py +++ b/sync_notion_to_chromadb.py @@ -9,10 +9,16 @@ load_dotenv() -NOTION_API_KEY = os.getenv("NOTION_API_KEY") -DATABASE_ID = "3538cd80a7f480aab786c93e0c370bf5" -STATE_FILE = "sync_state.json" -CHROMA_PATH = "./chroma_db" +STATE_FILE = os.getenv("SYNC_STATE_FILE", "sync_state.json") +CHROMA_PATH = os.getenv("CHROMA_PATH", "./chroma_db") +COLLECTION_NAME = os.getenv("BUSINESS_CASES_COLLECTION", "business_cases") + + +def get_notion_database_id(): + database_id = os.getenv("NOTION_DATABASE_ID") + if not database_id: + raise ValueError("NOTION_DATABASE_ID is required") + return database_id def safe_get_text(prop, default=""): if not prop: @@ -36,7 +42,7 @@ def safe_get_multi_select(prop): return [] ms = prop.get("multi_select", []) if isinstance(ms, list): - return [item.get("name", "") for item in ms if isinstance(item, dict)] + return [item.get("name", "") for item in ms if isinstance(item, dict) and item.get("name")] return [] def safe_get_url(prop, default=""): @@ -53,9 +59,9 @@ def safe_get_date(prop, default=""): return default def get_all_notion_pages(): - url = f"https://api.notion.com/v1/databases/{DATABASE_ID}/query" + url = f"https://api.notion.com/v1/databases/{get_notion_database_id()}/query" headers = { - "Authorization": f"Bearer {NOTION_API_KEY}", + "Authorization": f"Bearer {os.getenv('NOTION_API_KEY')}", "Content-Type": "application/json", "Notion-Version": "2022-06-28" } @@ -140,7 +146,7 @@ def sync(): model_name="text-embedding-3-small" ) collection = chroma_client.get_or_create_collection( - name="business_cases", + name=COLLECTION_NAME, embedding_function=openai_ef ) diff --git a/tests/test_memory.py b/tests/test_memory.py new file mode 100644 index 0000000..982ea58 --- /dev/null +++ b/tests/test_memory.py @@ -0,0 +1,72 @@ +import importlib +import sys +import types + + +def import_memory_with_fakes(monkeypatch): + upserts = [] + queries = [] + + fake_chromadb = types.ModuleType("chromadb") + fake_utils = types.ModuleType("chromadb.utils") + fake_embedding_functions = types.ModuleType("chromadb.utils.embedding_functions") + + class FakeOpenAIEmbeddingFunction: + def __init__(self, **kwargs): + self.kwargs = kwargs + + class FakeCollection: + def upsert(self, **kwargs): + upserts.append(kwargs) + + def query(self, **kwargs): + queries.append(kwargs) + return {"documents": [["previous answer", "older answer"]]} + + class FakeClient: + def __init__(self, path): + self.path = path + + def get_or_create_collection(self, **kwargs): + return FakeCollection() + + fake_embedding_functions.OpenAIEmbeddingFunction = FakeOpenAIEmbeddingFunction + fake_utils.embedding_functions = fake_embedding_functions + fake_chromadb.PersistentClient = FakeClient + + monkeypatch.setitem(sys.modules, "chromadb", fake_chromadb) + monkeypatch.setitem(sys.modules, "chromadb.utils", fake_utils) + monkeypatch.setitem(sys.modules, "chromadb.utils.embedding_functions", fake_embedding_functions) + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + monkeypatch.setenv("CHROMA_PATH", "/tmp/test-chroma") + monkeypatch.setenv("MEMORY_COLLECTION", "test-memory") + sys.modules.pop("memory", None) + + module = importlib.import_module("memory") + return module, upserts, queries + + +def test_add_to_memory_upserts_user_scoped_turn(monkeypatch): + memory, upserts, _ = import_memory_with_fakes(monkeypatch) + monkeypatch.setattr(memory.time, "time", lambda: 1234.567) + + memory.add_to_memory("user-1", "hello", "answer") + + assert upserts == [{ + "ids": ["user-1_1234567"], + "documents": ["Пользователь: hello\nАссистент: answer"], + "metadatas": [{"user_id": "user-1", "timestamp": 1234.567}], + }] + + +def test_retrieve_memory_filters_by_user(monkeypatch): + memory, _, queries = import_memory_with_fakes(monkeypatch) + + result = memory.retrieve_memory("user-1", "support automation", n_results=2) + + assert result == "previous answer\n\n---\n\nolder answer" + assert queries == [{ + "query_texts": ["support automation"], + "n_results": 2, + "where": {"user_id": "user-1"}, + }] diff --git a/tests/test_notion_to_chromadb.py b/tests/test_notion_to_chromadb.py index db33f16..4debd1c 100644 --- a/tests/test_notion_to_chromadb.py +++ b/tests/test_notion_to_chromadb.py @@ -51,6 +51,7 @@ def test_build_chroma_payload_formats_documents_and_metadata(): def test_fetch_notion_cases_paginates_and_maps_ru_fields(monkeypatch): + monkeypatch.setenv("NOTION_DATABASE_ID", "test-database-id") responses = [ { "results": [ diff --git a/tests/test_scribe.py b/tests/test_scribe.py index e859eb6..3b828a5 100644 --- a/tests/test_scribe.py +++ b/tests/test_scribe.py @@ -2,6 +2,8 @@ import sys import types +import pytest + def import_scribe(monkeypatch): fake_crewai = types.ModuleType("crewai") @@ -32,6 +34,7 @@ class FakeClient: pages = FakePages() monkeypatch.setenv("NOTION_API_KEY", "notion-token") + monkeypatch.setenv("NOTION_DATABASE_ID", "test-database-id") monkeypatch.setattr(scribe, "Client", lambda auth: FakeClient()) tool = scribe.NotionCreateCaseTool() @@ -50,7 +53,7 @@ class FakeClient: props = created["properties"] assert "успешно создан" in result - assert created["parent"] == {"database_id": "3538cd80a7f480aab786c93e0c370bf5"} + assert created["parent"] == {"database_id": "test-database-id"} assert props["Name"]["title"][0]["text"]["content"] == "Support Bot" assert props["Category"]["select"]["name"] == "Support" assert props["Tools"]["multi_select"] == [ @@ -60,6 +63,20 @@ class FakeClient: ] +def test_build_notion_case_properties_filters_empty_tools(monkeypatch): + scribe = import_scribe(monkeypatch) + + props = scribe.build_notion_case_properties({ + "title": "Case", + "category": "Automation", + "tools": "OpenAI, , ChromaDB", + }) + + assert props["Name"]["title"][0]["text"]["content"] == "Case" + assert props["Category"]["select"]["name"] == "Automation" + assert props["Tools"]["multi_select"] == [{"name": "OpenAI"}, {"name": "ChromaDB"}] + + def test_notion_create_case_tool_returns_error_on_notion_failure(monkeypatch): scribe = import_scribe(monkeypatch) @@ -71,6 +88,7 @@ class FakeClient: pages = FakePages() monkeypatch.setattr(scribe, "Client", lambda auth: FakeClient()) + monkeypatch.setenv("NOTION_DATABASE_ID", "test-database-id") tool = scribe.NotionCreateCaseTool() @@ -89,3 +107,11 @@ def _run(self, **kwargs): agent = scribe.ScribeAgent() assert agent.create_case({"title": "Case"}) == "created Case" + + +def test_notion_create_case_tool_requires_database_id(monkeypatch): + scribe = import_scribe(monkeypatch) + monkeypatch.delenv("NOTION_DATABASE_ID", raising=False) + + with pytest.raises(ValueError, match="NOTION_DATABASE_ID is required"): + scribe.NotionCreateCaseTool() diff --git a/tests/test_sync_notion_to_chromadb.py b/tests/test_sync_notion_to_chromadb.py index 9a6ef1f..484b52f 100644 --- a/tests/test_sync_notion_to_chromadb.py +++ b/tests/test_sync_notion_to_chromadb.py @@ -28,7 +28,7 @@ def test_safe_getters_handle_empty_values(): assert sync_module.safe_get_text({"rich_text": [{"text": {"content": "Body"}}]}) == "Body" assert sync_module.safe_get_select({"select": {"name": "Support"}}) == "Support" assert sync_module.safe_get_select({"select": None}) == "" - assert sync_module.safe_get_multi_select({"multi_select": [{"name": "A"}, {"bad": "ignored"}]}) == ["A", ""] + assert sync_module.safe_get_multi_select({"multi_select": [{"name": "A"}, {"bad": "ignored"}]}) == ["A"] assert sync_module.safe_get_multi_select({"multi_select": "not-a-list"}) == [] assert sync_module.safe_get_url({"url": "https://example.com"}) == "https://example.com" assert sync_module.safe_get_date({"date": {"start": "2026-05-23"}}) == "2026-05-23"