Add CineMatch RAG film recommendation system

.env.example

@@ -0,0 +1 @@
+OPENROUTER_API_KEY=your_key_here

.gitignore

@@ -0,0 +1,10 @@
+data/raw/
+.env
+__pycache__/
+chroma_db/
+*.pyc
+*.egg-info/
+.venv/
+venv/
+data/logs.db
+.DS_Store

config.yaml

@@ -0,0 +1,35 @@
+embedding_model: "all-MiniLM-L6-v2"
+reranker_model: "cross-encoder/ms-marco-MiniLM-L-6-v2"
+llm_model: "nvidia/nemotron-3-super-120b-a12b:free"
+fallback_models:
+  - "deepseek/deepseek-chat-v3-0324:free"
+  - "google/gemma-3-27b-it:free"
+  - "meta-llama/llama-4-maverick:free"
+openrouter_base_url: "https://openrouter.ai/api/v1"
+
+chroma_db_path: "chroma_db"
+chroma_collection: "movies"
+chroma_distance: "cosine"
+
+retrieval:
+  n_results: 20
+  top_k: 5
+  min_results_with_filter: 5
+  similarity_threshold: 0.4
+
+query_analyzer:
+  max_retries: 2
+  backoff_base_seconds: 2
+
+generation:
+  max_retries: 2
+  backoff_base_seconds: 2
+  history_turns: 2
+
+data:
+  raw_path: "data/raw"
+  processed_path: "data/processed"
+  movies_file: "data/processed/movies.jsonl"
+
+logging:
+  db_path: "data/logs.db"

data/test_queries.jsonl

@@ -0,0 +1,30 @@
+{"query": "Хочу что-то как Интерстеллар", "expected_genres": ["Science Fiction", "Drama"], "expected_type": "recommendation"}
+{"query": "Страшный фильм до 100 минут", "expected_genres": ["Horror"], "expected_type": "recommendation"}
+{"query": "Романтическая комедия для вечера", "expected_genres": ["Romance", "Comedy"], "expected_type": "recommendation"}
+{"query": "Эпический фэнтези фильм", "expected_genres": ["Fantasy", "Adventure"], "expected_type": "recommendation"}
+{"query": "Триллер с неожиданной концовкой", "expected_genres": ["Thriller"], "expected_type": "recommendation"}
+{"query": "Анимационный фильм для всей семьи", "expected_genres": ["Animation", "Family"], "expected_type": "recommendation"}
+{"query": "Военный фильм про вторую мировую", "expected_genres": ["War", "Drama"], "expected_type": "recommendation"}
+{"query": "Документальный фильм о природе", "expected_genres": ["Documentary"], "expected_type": "recommendation"}
+{"query": "Криминальная драма как Крёстный отец", "expected_genres": ["Crime", "Drama"], "expected_type": "recommendation"}
+{"query": "Фильм про супергероев", "expected_genres": ["Action", "Science Fiction"], "expected_type": "recommendation"}
+{"query": "Лёгкая комедия чтобы посмеяться", "expected_genres": ["Comedy"], "expected_type": "recommendation"}
+{"query": "Фильм о путешествии во времени", "expected_genres": ["Science Fiction"], "expected_type": "recommendation"}
+{"query": "Мюзикл с хорошими песнями", "expected_genres": ["Music"], "expected_type": "recommendation"}
+{"query": "Психологический триллер", "expected_genres": ["Thriller", "Drama"], "expected_type": "recommendation"}
+{"query": "Фильм-катастрофа", "expected_genres": ["Action", "Thriller"], "expected_type": "recommendation"}
+{"query": "Детективный фильм с загадкой", "expected_genres": ["Mystery", "Crime"], "expected_type": "recommendation"}
+{"query": "Спортивная драма", "expected_genres": ["Drama"], "expected_type": "recommendation"}
+{"query": "Фильм про космос и инопланетян", "expected_genres": ["Science Fiction"], "expected_type": "recommendation"}
+{"query": "Вестерн", "expected_genres": ["Western"], "expected_type": "recommendation"}
+{"query": "Фильм нуар", "expected_genres": ["Crime", "Thriller"], "expected_type": "recommendation"}
+{"query": "Биографический фильм о музыканте", "expected_genres": ["Drama", "Music"], "expected_type": "recommendation"}
+{"query": "Хороший фильм с рейтингом выше 8", "expected_genres": [], "expected_type": "recommendation"}
+{"query": "Новый фильм после 2015 года", "expected_genres": [], "expected_type": "recommendation"}
+{"query": "Короткий фильм до 90 минут", "expected_genres": [], "expected_type": "recommendation"}
+{"query": "Какая сегодня погода?", "expected_genres": [], "expected_type": "off_topic"}
+{"query": "Сколько будет 2+2?", "expected_genres": [], "expected_type": "off_topic"}
+{"query": "Расскажи анекдот", "expected_genres": [], "expected_type": "off_topic"}
+{"query": "Посоветуй что-то весёлое и доброе", "expected_genres": ["Comedy", "Family"], "expected_type": "recommendation"}
+{"query": "Мрачный фильм с глубоким смыслом", "expected_genres": ["Drama"], "expected_type": "recommendation"}
+{"query": "Приключенческий фильм для подростков", "expected_genres": ["Adventure"], "expected_type": "recommendation"}

prompts.yaml

@@ -0,0 +1,60 @@
+query_analyzer:
+  version: "1.0"
+  system: |
+    You are a movie recommendation query analyzer. Your task is to parse user queries (which may be in Russian or English) into a structured JSON format for a movie recommendation system.
+
+    IMPORTANT: The "semantic_query" field MUST be in English, as the movie database uses English embeddings.
+
+    If the query is NOT about movies or movie recommendations (e.g., weather, math, general questions), respond with:
+    {"off_topic": true}
+
+    Otherwise, respond with a JSON object:
+    {
+      "genre": "genre name or null",
+      "mood": "mood description or null",
+      "max_duration": integer or null (in minutes),
+      "min_year": integer or null,
+      "min_rating": float or null,
+      "semantic_query": "English search query for vector search"
+    }
+
+    Only output valid JSON, no explanations.
+
+  user_template: |
+    User query: {user_query}
+    Conversation context (last messages): {history}
+
+generation:
+  version: "1.0"
+  system: |
+    You are CineMatch, a friendly movie recommendation assistant. You recommend movies based on the provided search results.
+
+    RULES:
+    1. ONLY recommend movies from the provided list. NEVER invent or hallucinate movies.
+    2. Respond in the same language as the user's query.
+    3. For each movie, explain briefly WHY it matches the user's request.
+    4. If the results don't match well, say so honestly.
+    5. Keep responses concise and conversational.
+
+    Respond with a JSON object:
+    {
+      "movies": [
+        {
+          "title": "Movie Title",
+          "year": 2020,
+          "rating": 7.5,
+          "duration_min": 120,
+          "reason": "Brief explanation why this matches"
+        }
+      ],
+      "message": "Conversational intro/outro message in user's language"
+    }
+
+  user_template: |
+    User query: {user_query}
+
+    Retrieved movies:
+    {movies_json}
+
+    Conversation history:
+    {history}

requirements.txt

@@ -0,0 +1,9 @@
+streamlit>=1.30.0
+chromadb>=0.4.22
+sentence-transformers>=2.3.0
+openai>=1.0.0
+kagglehub>=0.2.0
+pandas>=2.1.0
+pyyaml>=6.0
+python-dotenv>=1.0.0
+scikit-learn>=1.3.0

scripts/build_index.py

@@ -0,0 +1,101 @@
+"""Build ChromaDB vector index from processed movies.jsonl."""
+
+import json
+import sys
+from pathlib import Path
+
+import chromadb
+from sentence_transformers import SentenceTransformer
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(PROJECT_ROOT))
+
+import yaml
+
+with open(PROJECT_ROOT / "config.yaml") as f:
+    config = yaml.safe_load(f)
+
+MOVIES_FILE = PROJECT_ROOT / config["data"]["movies_file"]
+CHROMA_PATH = PROJECT_ROOT / config["chroma_db_path"]
+COLLECTION_NAME = config["chroma_collection"]
+EMBEDDING_MODEL = config["embedding_model"]
+
+
+def load_movies(path: Path) -> list[dict]:
+    movies = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            movies.append(json.loads(line))
+    return movies
+
+
+def build_index():
+    print(f"Loading movies from {MOVIES_FILE}...")
+    movies = load_movies(MOVIES_FILE)
+    print(f"Loaded {len(movies)} movies")
+
+    print(f"Loading embedding model: {EMBEDDING_MODEL}...")
+    model = SentenceTransformer(EMBEDDING_MODEL)
+
+    texts = [m["text_for_embedding"] for m in movies]
+    print(f"Generating embeddings for {len(texts)} documents (batch_size=64)...")
+    embeddings = model.encode(texts, batch_size=64, show_progress_bar=True)
+    print(f"Embeddings shape: {embeddings.shape}")
+
+    print(f"Creating ChromaDB at {CHROMA_PATH}...")
+    client = chromadb.PersistentClient(path=str(CHROMA_PATH))
+
+    try:
+        client.delete_collection(COLLECTION_NAME)
+        print(f"Deleted existing collection '{COLLECTION_NAME}'")
+    except Exception:
+        pass
+
+    collection = client.create_collection(
+        name=COLLECTION_NAME,
+        metadata={"hnsw:space": "cosine"},
+    )
+
+    batch_size = 500
+    for i in range(0, len(movies), batch_size):
+        batch = movies[i : i + batch_size]
+        batch_embeddings = embeddings[i : i + batch_size].tolist()
+
+        ids = [m["id"] for m in batch]
+        documents = [m["text_for_embedding"] for m in batch]
+        metadatas = [
+            {
+                "title": m["title"],
+                "year": int(m["year"]),
+                "duration_min": int(m["duration_min"]),
+                "rating": float(m["rating"]),
+                "genres": m["genres_pipe"],
+                "tags": m["tags_pipe"],
+                "overview": m["overview"],
+            }
+            for m in batch
+        ]
+
+        collection.add(
+            ids=ids,
+            embeddings=batch_embeddings,
+            documents=documents,
+            metadatas=metadatas,
+        )
+        print(f"  Added batch {i // batch_size + 1}: {len(batch)} documents")
+
+    print(f"Total documents in collection: {collection.count()}")
+
+    print("\nTest query: 'space exploration emotional drama'")
+    results = collection.query(
+        query_embeddings=model.encode(["space exploration emotional drama"]).tolist(),
+        n_results=5,
+    )
+    for i, (doc_id, metadata) in enumerate(zip(results["ids"][0], results["metadatas"][0])):
+        print(f"  {i + 1}. {metadata['title']} ({metadata['year']}) - rating: {metadata['rating']}")
+
+    print("\nIndex built successfully!")
+
+
+if __name__ == "__main__":
+    build_index()