From ab6bcd6ee07a59eed5f7daf73e174b3be5218066 Mon Sep 17 00:00:00 2001 From: andreikurakin Date: Wed, 1 Apr 2026 22:52:37 +0300 Subject: [PATCH 1/5] make parser and norm text, need add link to file --- parser/parser.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 parser/parser.py diff --git a/parser/parser.py b/parser/parser.py new file mode 100644 index 0000000..064a3ef --- /dev/null +++ b/parser/parser.py @@ -0,0 +1,74 @@ +import pdfplumber +import re +import pymorphy2 + +def parse_hall_text(pdf_pass): + + result = [] + + with pdfplumber.open(pdf_pass) as pdf: + for page_num, page in enumerate(pdf.pages, 1): + text = page.extract_text() + + result.append({ + 'page': page_num, + 'text': text + }) + + return result + +def filter_text(text): + text = re.sub(r'\n+', '\n', text) + + text = re.sub(r'\s+', ' ', text) + + text = re.sub(r'\(стр\.\s+\d+\)', '', text, flags=re.IGNORECASE) + text = re.sub(r'\[страница\s+\d+\]', '', text, flags=re.IGNORECASE) + + # 5. Удаляем URL + text = re.sub(r'https?://\S+', '', text) + + # 6. Финальная очистка пробелов + text = text.strip() + text = re.sub(r'\s+', ' ', text) + + return text + +morph = pymorphy2.MorphAnalyzer() + +def normalize_text_lemmatize(text, language='ru'): + + text = text.lower() + + words = text.split() + lemmatized = [] + + for word in words: + parsed = morph.parse(word)[0] + lemma = parsed.normal_form + lemmatized.append(lemma.normal_form) + + return ' '.join(lemmatized) + + +def prepare_text_for_rag(pdf_path): + + chunks = parse_hall_text(pdf_path) + + result = [] + for chunk in chunks: + + filtered = filter_text(chunk['text']) + + normalized = normalize_text_lemmatize(filtered) + + result.append({ + "page": chunk['page'], + "original": chunk['text'], + "normalized": normalized + }) + + return result + +if __name__ == "__main__": + pdf_pass = "" \ No newline at end of file From 885b7e6a53f8c0da822d7a00324ba694d79555e0 Mon Sep 17 00:00:00 2001 From: andreikurakin Date: Fri, 3 Apr 2026 23:33:59 +0300 Subject: [PATCH 2/5] add parser --- parser/main.py | 50 ++++++++ parser/parser.py | 275 ++++++++++++++++++++++++++++++---------- parser/pyproject.toml | 11 ++ parser/requirements.txt | 3 +- 4 files changed, 272 insertions(+), 67 deletions(-) create mode 100644 parser/main.py create mode 100644 parser/pyproject.toml diff --git a/parser/main.py b/parser/main.py new file mode 100644 index 0000000..d635583 --- /dev/null +++ b/parser/main.py @@ -0,0 +1,50 @@ +from pathlib import Path +from sentence_transformers import SentenceTransformer +from parser import prepare_text_for_rag, generate_sql + + +def main(): + print( + "⏳ Инициализация модели " + "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2..." + ) + # Сохраняем модель локально в /parser/models/, как сказано в ТЗ: + models_dir = Path(__file__).parent / "models" + models_dir.mkdir(parents=True, exist_ok=True) + try: + model = SentenceTransformer( + "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + cache_folder=str(models_dir), + ) + except Exception as e: + print(f"❌ Ошибка при загрузке модели: {e}") + return + + data_dir = Path(__file__).parent.parent / "data" + if not data_dir.exists(): + print( + f"⚠️ Папка {data_dir} не найдена. " + "Создаю пустую папку (положите в нее PDF-файлы)." + ) + data_dir.mkdir(parents=True, exist_ok=True) + + pdf_files = list(data_dir.glob("*.pdf")) + if not pdf_files: + print(f"⚠️ В папке {data_dir} нет PDF файлов.") + return + + print(f"🔍 Найдено {len(pdf_files)} PDF файлов для обработки.") + + def all_pdfs_generator(): + for pdf_path in pdf_files: + print(f"📖 Читаем документ: {pdf_path.name}") + # Извлекаем title / chapter_name из имени файла (к примеру) + chapter_name = pdf_path.stem + yield from prepare_text_for_rag(pdf_path, model, chapter_name=chapter_name) + + output_sql = Path(__file__).parent / "init.sql" + generate_sql(all_pdfs_generator(), output_file=str(output_sql)) + + +if __name__ == "__main__": + main() diff --git a/parser/parser.py b/parser/parser.py index 064a3ef..6730fbc 100644 --- a/parser/parser.py +++ b/parser/parser.py @@ -1,74 +1,217 @@ -import pdfplumber import re -import pymorphy2 +import pdfplumber +import yake + -def parse_hall_text(pdf_pass): - - result = [] - - with pdfplumber.open(pdf_pass) as pdf: +def parse_hall_text(pdf_path, start_page=1, end_page=None): + """ + Извлекает текст из PDF разными методами. + Возвращает генератор объектов {page, text}. + """ + with pdfplumber.open(pdf_path) as pdf: for page_num, page in enumerate(pdf.pages, 1): + if page_num < start_page: + continue + if end_page and page_num > end_page: + break + text = page.extract_text() - - result.append({ - 'page': page_num, - 'text': text - }) - - return result + + if not text or len(text.strip()) < 10: + try: + text = page.extract_text(x_tolerance=2, y_tolerance=2) + except Exception: + pass + + if not text or len(text.strip()) < 10: + tables = page.extract_tables() + if tables: + text = "\n".join( + [ + "\n".join([str(cell) for cell in row]) + for table in tables + for row in table + ] + ) + + if text and len(text.strip()) > 10: + yield { + "page": page_num, + "text": text, + } + else: + print(f"⚠️ Страница {page_num}: текст не найден") + + page.flush_cache() + + if page_num % 10 == 0: + print(f"⌛ Обработано {page_num} страниц...") + def filter_text(text): - text = re.sub(r'\n+', '\n', text) - - text = re.sub(r'\s+', ' ', text) - - text = re.sub(r'\(стр\.\s+\d+\)', '', text, flags=re.IGNORECASE) - text = re.sub(r'\[страница\s+\d+\]', '', text, flags=re.IGNORECASE) - - # 5. Удаляем URL - text = re.sub(r'https?://\S+', '', text) - - # 6. Финальная очистка пробелов - text = text.strip() - text = re.sub(r'\s+', ' ', text) - - return text - -morph = pymorphy2.MorphAnalyzer() - -def normalize_text_lemmatize(text, language='ru'): - - text = text.lower() - - words = text.split() - lemmatized = [] - - for word in words: - parsed = morph.parse(word)[0] - lemma = parsed.normal_form - lemmatized.append(lemma.normal_form) - - return ' '.join(lemmatized) - - -def prepare_text_for_rag(pdf_path): - - chunks = parse_hall_text(pdf_path) - - result = [] + """Фильтрует текст от мусора и служебной информации.""" + text = re.sub( + r"---algorithm---.*?---end---", "", text, flags=re.DOTALL | re.IGNORECASE + ) + text = re.sub(r"(?:рис|fig)\.\s*\d+[^\n]*", "", text, flags=re.IGNORECASE) + text = re.sub(r"\(стр\.\s+\d+\)", "", text, flags=re.IGNORECASE) + text = re.sub(r"\[страница\s+\d+\]", "", text, flags=re.IGNORECASE) + text = re.sub(r"https?://\S+", "", text) + text = re.sub(r"\n+", "\n", text) + text = re.sub(r"\s+", " ", text) + return text.strip() + + +def extract_keywords(text, top_n=20): + """Извлечение ключевых слов через алгоритм YAKE.""" + # Оффлайн NLP извлечение терминов, отсекаются общие/стоп слова + kw_extractor = yake.KeywordExtractor( + lan="ru", n=1, dedupLim=0.9, top=top_n, features=None + ) + keywords = kw_extractor.extract_keywords(text) + return [kw[0] for kw in keywords] + + +def split_into_blocks(text): + """ + Разбивает текст на логические блоки (Определения, Теоремы и обычный текст). + """ + blocks = [] + current_type = "general" + current_text = [] + + sentences = re.split(r"(?<=[.!?])\s+", text) + + for sentence in sentences: + lower_s = sentence.lower() + + if lower_s.startswith("определение"): + if current_text: + blocks.append({"type": current_type, "text": " ".join(current_text)}) + current_type = "definition" + current_text = [sentence] + elif lower_s.startswith("теорема") or lower_s.startswith("лемма"): + if current_text: + blocks.append({"type": current_type, "text": " ".join(current_text)}) + current_type = "theorem" + current_text = [sentence] + else: + current_text.append(sentence) + + if current_text: + blocks.append({"type": current_type, "text": " ".join(current_text)}) + + return blocks + + +def prepare_text_for_rag( + pdf_path, model, chapter_name="unknown", start_page=1, end_page=None +): + """ + Полный pipeline для RAG: парсинг → фильтрация → блоки → эмбеддинги. + """ + chunks = parse_hall_text(pdf_path, start_page, end_page) + for chunk in chunks: + cleaned = filter_text(chunk["text"]) + if not cleaned.strip(): + continue + + blocks = split_into_blocks(cleaned) + + for block in blocks: + block_text = block["text"] + if not block_text.strip(): + continue + + keywords = extract_keywords(block_text) + embedding = model.encode(block_text).tolist() + + yield { + "chapter_name": chapter_name, + "keywords": keywords, + "chunk_text": block_text, + "embedding": embedding, + } + + +def escape_sql_string(s): + """Помощник для эскейпинга одинарных кавычек в SQL.""" + return s.replace("'", "''") + + +INSERT_VALUES_PER_STATEMENT = 50 + + +def format_insert_values_row(item): + """Одна строка для списка VALUES (...), (...), ...""" + kw_cleaned = [ + str(kw).replace("'", "").replace('"', "") for kw in item["keywords"][:10] + ] + keywords_pg = "{" + ",".join(kw_cleaned) + "}" + embedding_str = "[" + ",".join(map(str, item["embedding"])) + "]" + return ( + f"('{escape_sql_string(item['chapter_name'])}', " + f"'{keywords_pg}', " + f"'{escape_sql_string(item['chunk_text'])}', " + f"'{embedding_str}')" + ) + + +def generate_sql(data_generator, output_file="init.sql"): + """ + Генерирует init.sql: DDL и пакетные INSERT (не по одной строке на команду). + """ + init_script = [ + "-- Инициализация таблиц для RAG", + "CREATE EXTENSION IF NOT EXISTS vector;", + "CREATE TABLE IF NOT EXISTS documents (", + " id SERIAL PRIMARY KEY,", + " chapter_name TEXT,", + " keywords TEXT[],", + " chunk_text TEXT,", + " embedding vector(384)", + ");", + "", + f"-- Данные: один INSERT содержит до {INSERT_VALUES_PER_STATEMENT} строк.", + "", + ] + + insert_header = ( + "INSERT INTO documents (chapter_name, keywords, chunk_text, embedding) VALUES\n" + ) + + with open(output_file, "w", encoding="utf-8") as f: + for line in init_script: + f.write(line + "\n") + + count = 0 + batch = [] + statements = 0 + + def flush_batch(): + nonlocal batch, statements + if not batch: + return + f.write(insert_header) + f.write(",\n".join(batch)) + f.write(";\n\n") + statements += 1 + batch = [] + + for item in data_generator: + count += 1 + batch.append(format_insert_values_row(item)) + + if len(batch) >= INSERT_VALUES_PER_STATEMENT: + flush_batch() + + if count % 10 == 0: + print(f"✍️ Обработано и записано {count} логических блоков в SQL...") + + flush_batch() - filtered = filter_text(chunk['text']) - - normalized = normalize_text_lemmatize(filtered) - - result.append({ - "page": chunk['page'], - "original": chunk['text'], - "normalized": normalized - }) - - return result - -if __name__ == "__main__": - pdf_pass = "" \ No newline at end of file + print( + f"✅ SQL скрипт успешно сгенерирован: {output_file}. " + f"Всего записей (блоков): {count}, операторов INSERT: {statements}" + ) diff --git a/parser/pyproject.toml b/parser/pyproject.toml new file mode 100644 index 0000000..562c548 --- /dev/null +++ b/parser/pyproject.toml @@ -0,0 +1,11 @@ +[tool.ruff] +line-length = 88 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "F", "B"] +ignore = [] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" diff --git a/parser/requirements.txt b/parser/requirements.txt index feb7b9a..c4165eb 100644 --- a/parser/requirements.txt +++ b/parser/requirements.txt @@ -1,4 +1,5 @@ pdfplumber sentence-transformers onnxruntime -psycopg2-binary \ No newline at end of file +ruff +yake From af2e16c53bf27c8a294e21af0470cf634a442d81 Mon Sep 17 00:00:00 2001 From: andreikurakin Date: Thu, 9 Apr 2026 20:22:01 +0300 Subject: [PATCH 3/5] delete smiles, make new comment, add new parser logic --- parser/extractors/__init__.py | 4 + parser/extractors/pdf_extractor.py | 61 +++++++ parser/generators/__init__.py | 4 + parser/generators/sql_generator.py | 149 +++++++++++++++++ parser/main.py | 29 ++-- parser/nlp/__init__.py | 5 + parser/nlp/keyword_extractor.py | 83 ++++++++++ parser/nlp/text_processor.py | 71 ++++++++ parser/parser.py | 250 +++++++---------------------- parser/requirements.txt | 1 + 10 files changed, 453 insertions(+), 204 deletions(-) create mode 100644 parser/extractors/__init__.py create mode 100644 parser/extractors/pdf_extractor.py create mode 100644 parser/generators/__init__.py create mode 100644 parser/generators/sql_generator.py create mode 100644 parser/nlp/__init__.py create mode 100644 parser/nlp/keyword_extractor.py create mode 100644 parser/nlp/text_processor.py diff --git a/parser/extractors/__init__.py b/parser/extractors/__init__.py new file mode 100644 index 0000000..5cc1457 --- /dev/null +++ b/parser/extractors/__init__.py @@ -0,0 +1,4 @@ +"""PDF extraction and text processing module.""" +from .pdf_extractor import parse_hall_text + +__all__ = ["parse_hall_text"] diff --git a/parser/extractors/pdf_extractor.py b/parser/extractors/pdf_extractor.py new file mode 100644 index 0000000..5fbde83 --- /dev/null +++ b/parser/extractors/pdf_extractor.py @@ -0,0 +1,61 @@ +"""PDF text extraction module.""" +from typing import Generator, Dict, Optional +import pdfplumber + + +def parse_hall_text( + pdf_path: str, start_page: int = 1, end_page: Optional[int] = None +) -> Generator[Dict[str, object], None, None]: + """Extract text from PDF using multiple fallback methods. + + Attempts to extract text from each page using pdfplumber. If initial extraction + fails or produces insufficient text, falls back to adjusted tolerance settings + and table extraction. + + Args: + pdf_path: Path to the PDF file. + start_page: Starting page number (1-indexed). Defaults to 1. + end_page: Ending page number (inclusive). If None, processes all pages. + + Yields: + Dictionary containing 'page' (int) and 'text' (str) keys. + Only yields pages with extracted text length > 10 characters. + """ + with pdfplumber.open(pdf_path) as pdf: + for page_num, page in enumerate(pdf.pages, 1): + if page_num < start_page: + continue + if end_page and page_num > end_page: + break + + text = page.extract_text() + + if not text or len(text.strip()) < 10: + try: + text = page.extract_text(x_tolerance=2, y_tolerance=2) + except Exception: + pass + + if not text or len(text.strip()) < 10: + tables = page.extract_tables() + if tables: + text = "\n".join( + [ + "\n".join([str(cell) for cell in row]) + for table in tables + for row in table + ] + ) + + if text and len(text.strip()) > 10: + yield { + "page": page_num, + "text": text, + } + else: + print(f"Warning: Page {page_num} - no text extracted") + + page.flush_cache() + + if page_num % 10 == 0: + print(f"Processed {page_num} pages...") diff --git a/parser/generators/__init__.py b/parser/generators/__init__.py new file mode 100644 index 0000000..3762352 --- /dev/null +++ b/parser/generators/__init__.py @@ -0,0 +1,4 @@ +"""SQL generation and data mapping module.""" +from .sql_generator import generate_sql, escape_sql_string, format_insert_values_rows + +__all__ = ["generate_sql", "escape_sql_string", "format_insert_values_rows"] diff --git a/parser/generators/sql_generator.py b/parser/generators/sql_generator.py new file mode 100644 index 0000000..c0caebe --- /dev/null +++ b/parser/generators/sql_generator.py @@ -0,0 +1,149 @@ +"""SQL generation and data formatting for database initialization.""" +from typing import Dict, Any, List + +INSERT_VALUES_PER_STATEMENT = 50 + + +def escape_sql_string(s: str) -> str: + """Escape single quotes in SQL string values. + + Args: + s: String to escape. + + Returns: + String with single quotes escaped for SQL. + """ + return s.replace("'", "''") + + +def format_insert_values_rows(item: Dict[str, Any]) -> List[str]: + """Format data items as multiple SQL VALUES rows (one per term). + + Converts extracted terms and their embeddings into individual card_contents + records. Each term becomes a separate flashcard with its definition and + embedding. + + Args: + item: Dictionary with keys: chapter_name, terms, definitions, + chunk_text, terms_embeddings. + + Returns: + List of SQL VALUES rows, one per extracted term. + """ + rows = [] + + # Iterate through terms and create a card for each + for i, (term, definition, embedding) in enumerate( + zip(item["terms"], item["definitions"], item["terms_embeddings"]) + ): + # Format embedding as PostgreSQL vector + embedding_str = "[" + ",".join(map(str, embedding)) + "]" + + # Create source info: chapter + chunk context + source_info = f"{item['chapter_name']}|{item['chunk_text'][:200]}" + + row = ( + f"('{escape_sql_string(term)}', " + f"'{escape_sql_string(definition[:500])}', " + f"'{embedding_str}', " + f"'{escape_sql_string(source_info)}')" + ) + rows.append(row) + + return rows + + +def generate_sql( + data_generator: Any, output_file: str = "init.sql" +) -> None: + """Generate SQL initialization script with DDL and batch INSERT statements. + + Creates a PostgreSQL initialization script containing table creation + (with vector extension) and batch INSERT statements for card_contents and + card_progress tables. Batches are sized for optimal performance. + + Args: + data_generator: Generator/iterable yielding data dictionaries. + output_file: Path where SQL script will be written. Defaults to 'init.sql'. + + Returns: + None. Writes directly to output_file. + """ + init_script = [ + "-- RAG Flashcard Schema", + "CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";", + "CREATE EXTENSION IF NOT EXISTS vector;", + "", + "-- Table: card_contents", + "-- Stores individual terms with embeddings and source context", + "CREATE TABLE IF NOT EXISTS card_contents (", + " id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),", + " front_text TEXT NOT NULL,", + " back_text TEXT NOT NULL,", + " embedding vector(384),", + " source_info TEXT,", + " created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()", + ");", + "", + "-- HNSW index for similarity search", + "CREATE INDEX ON card_contents USING hnsw (embedding vector_cosine_ops);", + "", + "-- Table: card_progress", + "-- Tracks spaced repetition metrics for each card", + "CREATE TABLE IF NOT EXISTS card_progress (", + " id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),", + " card_id UUID NOT NULL UNIQUE REFERENCES card_contents (id) ON DELETE CASCADE,", + " interval BIGINT NOT NULL DEFAULT 0,", + " easiness REAL NOT NULL DEFAULT 2.5,", + " repetitions INT NOT NULL DEFAULT 0,", + " next_review_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),", + " updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()", + ");", + "", + "CREATE INDEX IF NOT EXISTS idx_progress_next_review ON card_progress (next_review_at);", + "", + f"-- Data: each INSERT contains up to {INSERT_VALUES_PER_STATEMENT} rows.", + "", + ] + + insert_header = ( + "INSERT INTO card_contents (front_text, back_text, embedding, source_info) VALUES\n" + ) + + with open(output_file, "w", encoding="utf-8") as f: + for line in init_script: + f.write(line + "\n") + + count = 0 + batch = [] + statements = 0 + + def flush_batch(): + nonlocal batch, statements + if not batch: + return + f.write(insert_header) + f.write(",\n".join(batch)) + f.write(";\n\n") + statements += 1 + batch = [] + + for item in data_generator: + # Each item can produce multiple rows (one per term) + rows = format_insert_values_rows(item) + for row in rows: + count += 1 + batch.append(row) + + if len(batch) >= INSERT_VALUES_PER_STATEMENT: + flush_batch() + + if count % 100 == 0: + print(f"Processed and written {count} flashcards to SQL...") + + flush_batch() + + print( + f"SQL script generated successfully: {output_file}. " + f"Total flashcards: {count}, INSERT statements: {statements}" + ) diff --git a/parser/main.py b/parser/main.py index d635583..53e3e7c 100644 --- a/parser/main.py +++ b/parser/main.py @@ -1,14 +1,18 @@ +"""Entry point for RAG document processing pipeline.""" from pathlib import Path +from typing import Generator, Dict, Any from sentence_transformers import SentenceTransformer -from parser import prepare_text_for_rag, generate_sql +from parser import prepare_text_for_rag +from generators import generate_sql -def main(): +def main() -> None: + """Initialize model and execute RAG pipeline on all PDF files in data directory.""" print( - "⏳ Инициализация модели " + "Initializing model " "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2..." ) - # Сохраняем модель локально в /parser/models/, как сказано в ТЗ: + # Cache models locally in /parser/models/ models_dir = Path(__file__).parent / "models" models_dir.mkdir(parents=True, exist_ok=True) try: @@ -17,28 +21,29 @@ def main(): cache_folder=str(models_dir), ) except Exception as e: - print(f"❌ Ошибка при загрузке модели: {e}") + print(f"Error loading model: {e}") return data_dir = Path(__file__).parent.parent / "data" if not data_dir.exists(): print( - f"⚠️ Папка {data_dir} не найдена. " - "Создаю пустую папку (положите в нее PDF-файлы)." + f"Data directory {data_dir} not found. " + "Creating empty directory (add PDF files to it)." ) data_dir.mkdir(parents=True, exist_ok=True) pdf_files = list(data_dir.glob("*.pdf")) if not pdf_files: - print(f"⚠️ В папке {data_dir} нет PDF файлов.") + print(f"No PDF files found in {data_dir}.") return - print(f"🔍 Найдено {len(pdf_files)} PDF файлов для обработки.") + print(f"Found {len(pdf_files)} PDF files to process.") - def all_pdfs_generator(): + def all_pdfs_generator() -> Generator[Dict[str, Any], None, None]: + """Generate processed data from all PDF files.""" for pdf_path in pdf_files: - print(f"📖 Читаем документ: {pdf_path.name}") - # Извлекаем title / chapter_name из имени файла (к примеру) + print(f"Processing document: {pdf_path.name}") + # Extract chapter name from filename chapter_name = pdf_path.stem yield from prepare_text_for_rag(pdf_path, model, chapter_name=chapter_name) diff --git a/parser/nlp/__init__.py b/parser/nlp/__init__.py new file mode 100644 index 0000000..2553d95 --- /dev/null +++ b/parser/nlp/__init__.py @@ -0,0 +1,5 @@ +"""NLP processing module for text cleaning and keyword extraction.""" +from .text_processor import filter_text, split_into_blocks +from .keyword_extractor import extract_keywords + +__all__ = ["filter_text", "split_into_blocks", "extract_keywords"] diff --git a/parser/nlp/keyword_extractor.py b/parser/nlp/keyword_extractor.py new file mode 100644 index 0000000..74f30b5 --- /dev/null +++ b/parser/nlp/keyword_extractor.py @@ -0,0 +1,83 @@ +"""Keyword extraction using YAKE algorithm.""" +from typing import List, Tuple, Set +import string +import yake + +# Russian stop words for filtering +RU_STOP_WORDS: Set[str] = { + 'что', 'это', 'как', 'за', 'из', 'на', 'при', 'в', 'с', 'по', 'к', 'до', + 'и', 'или', 'но', 'а', 'не', 'ни', 'да', 'нет', 'если', 'то', + 'для', 'от', 'об', 'о', 'без', 'через', 'между', 'перед', 'после', + 'выше', 'ниже', 'раньше', 'позже', 'всегда', 'никогда', 'часто', 'редко', + 'который', 'какой', 'какая', 'какое', 'кто', 'где', 'когда', 'почему', 'зачем', + 'я', 'ты', 'он', 'она', 'оно', 'мы', 'вы', 'они', 'ме', 'те', 'же', 'ее', 'его', + 'более', 'менее', 'очень', 'же', 'же', +} + + +def extract_keywords(text: str, top_n: int = 20) -> List[Tuple[str, str]]: + """Extract and filter keywords/terms from text. + + Uses YAKE algorithm to identify single and two-word terms, then filters + based on Russian stop words, minimum length, and stop word ratio. + Original word forms are preserved while normalized versions are used for + embedding generation. + + Args: + text: Input text to extract keywords from. + top_n: Maximum number of keywords to return. Defaults to 20. + + Returns: + List of tuples (original_word, normalized_word) for each extracted keyword. + Normalized words are lowercase for consistent embedding generation. + """ + if not text.strip(): + return [] + + # Extract single and two-word terms using YAKE + try: + kw_extractor = yake.KeywordExtractor( + lan="ru", n=2, dedupLim=0.9, top=top_n*2, features=None + ) + keywords_raw = kw_extractor.extract_keywords(text) + except Exception: + return [] + + filtered_keywords: List[Tuple[str, str]] = [] + seen_normalized: Set[str] = set() + + for kw, score in keywords_raw: + kw_original = kw.strip() # Preserve original word form + kw_normalized = kw_original.lower() # Normalize for filtering + + # Skip if empty or exact match to stop word + if not kw_normalized or kw_normalized in RU_STOP_WORDS: + continue + + # Check minimum length (at least 2 characters) + if len(kw_normalized) < 2: + continue + + # Remove words containing only punctuation or digits + if all(c in string.punctuation + string.digits for c in kw_normalized): + continue + + # Remove single-word terms that are stop words + words = kw_normalized.split() + if all(w in RU_STOP_WORDS for w in words): + continue + + # Remove phrases where more than half are stop words + stop_word_ratio = sum(1 for w in words if w in RU_STOP_WORDS) / len(words) + if stop_word_ratio > 0.5: + continue + + # Remove duplicates by normalized form + if kw_normalized not in seen_normalized: + filtered_keywords.append((kw_original, kw_normalized)) + seen_normalized.add(kw_normalized) + + if len(filtered_keywords) >= top_n: + break + + return filtered_keywords diff --git a/parser/nlp/text_processor.py b/parser/nlp/text_processor.py new file mode 100644 index 0000000..acf83fd --- /dev/null +++ b/parser/nlp/text_processor.py @@ -0,0 +1,71 @@ + +"""Text cleaning and block splitting module.""" +from typing import List, Dict +import re + + +def filter_text(text: str) -> str: + """Remove noise and service information from text. + + Removes algorithm blocks, figure/table references, page numbers, URLs, + email addresses, excess whitespace, and hanging punctuation. + + Args: + text: Raw text to clean. + + Returns: + Cleaned text with all filtered elements removed. + """ + # Remove algorithm blocks + text = re.sub( + r"---algorithm---.*?---end---", "", text, flags=re.DOTALL | re.IGNORECASE + ) + # Remove figure and table references + text = re.sub(r"(?:рис|fig)\.\s*\d+[^\n]*", "", text, flags=re.IGNORECASE) + text = re.sub(r"(?:табл|table)\.\s*\d+[^\n]*", "", text, flags=re.IGNORECASE) + # Remove page numbers and service elements + text = re.sub(r"\(стр\.\s+\d+\)", "", text, flags=re.IGNORECASE) + text = re.sub(r"\[страница\s+\d+\]", "", text, flags=re.IGNORECASE) + text = re.sub(r"^-+\s*\d+\s*-+$", "", text, flags=re.MULTILINE) + # Remove URLs + text = re.sub(r"https?://\S+", "", text) + # Remove email addresses + text = re.sub(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", "", text) + # Remove excess whitespace and line breaks + text = re.sub(r"\n+", "\n", text) + text = re.sub(r"\s+", " ", text) + # Remove hanging punctuation + text = re.sub(r"^\s*[^\w\s]*\s+", "", text, flags=re.MULTILINE) + return text.strip() + + +def split_into_blocks(text): + """ + Разбивает текст на логические блоки (Определения, Теоремы и обычный текст). + """ + blocks = [] + current_type = "general" + current_text = [] + + sentences = re.split(r"(?<=[.!?])\s+", text) + + for sentence in sentences: + lower_s = sentence.lower() + + if lower_s.startswith("определение"): + if current_text: + blocks.append({"type": current_type, "text": " ".join(current_text)}) + current_type = "definition" + current_text = [sentence] + elif lower_s.startswith("теорема") or lower_s.startswith("лемма"): + if current_text: + blocks.append({"type": current_type, "text": " ".join(current_text)}) + current_type = "theorem" + current_text = [sentence] + else: + current_text.append(sentence) + + if current_text: + blocks.append({"type": current_type, "text": " ".join(current_text)}) + + return blocks diff --git a/parser/parser.py b/parser/parser.py index 6730fbc..deeebb2 100644 --- a/parser/parser.py +++ b/parser/parser.py @@ -1,114 +1,41 @@ -import re -import pdfplumber -import yake - - -def parse_hall_text(pdf_path, start_page=1, end_page=None): - """ - Извлекает текст из PDF разными методами. - Возвращает генератор объектов {page, text}. - """ - with pdfplumber.open(pdf_path) as pdf: - for page_num, page in enumerate(pdf.pages, 1): - if page_num < start_page: - continue - if end_page and page_num > end_page: - break - - text = page.extract_text() - - if not text or len(text.strip()) < 10: - try: - text = page.extract_text(x_tolerance=2, y_tolerance=2) - except Exception: - pass - - if not text or len(text.strip()) < 10: - tables = page.extract_tables() - if tables: - text = "\n".join( - [ - "\n".join([str(cell) for cell in row]) - for table in tables - for row in table - ] - ) - - if text and len(text.strip()) > 10: - yield { - "page": page_num, - "text": text, - } - else: - print(f"⚠️ Страница {page_num}: текст не найден") - - page.flush_cache() - - if page_num % 10 == 0: - print(f"⌛ Обработано {page_num} страниц...") - - -def filter_text(text): - """Фильтрует текст от мусора и служебной информации.""" - text = re.sub( - r"---algorithm---.*?---end---", "", text, flags=re.DOTALL | re.IGNORECASE - ) - text = re.sub(r"(?:рис|fig)\.\s*\d+[^\n]*", "", text, flags=re.IGNORECASE) - text = re.sub(r"\(стр\.\s+\d+\)", "", text, flags=re.IGNORECASE) - text = re.sub(r"\[страница\s+\d+\]", "", text, flags=re.IGNORECASE) - text = re.sub(r"https?://\S+", "", text) - text = re.sub(r"\n+", "\n", text) - text = re.sub(r"\s+", " ", text) - return text.strip() - - -def extract_keywords(text, top_n=20): - """Извлечение ключевых слов через алгоритм YAKE.""" - # Оффлайн NLP извлечение терминов, отсекаются общие/стоп слова - kw_extractor = yake.KeywordExtractor( - lan="ru", n=1, dedupLim=0.9, top=top_n, features=None - ) - keywords = kw_extractor.extract_keywords(text) - return [kw[0] for kw in keywords] - - -def split_into_blocks(text): - """ - Разбивает текст на логические блоки (Определения, Теоремы и обычный текст). - """ - blocks = [] - current_type = "general" - current_text = [] - - sentences = re.split(r"(?<=[.!?])\s+", text) - - for sentence in sentences: - lower_s = sentence.lower() - - if lower_s.startswith("определение"): - if current_text: - blocks.append({"type": current_type, "text": " ".join(current_text)}) - current_type = "definition" - current_text = [sentence] - elif lower_s.startswith("теорема") or lower_s.startswith("лемма"): - if current_text: - blocks.append({"type": current_type, "text": " ".join(current_text)}) - current_type = "theorem" - current_text = [sentence] - else: - current_text.append(sentence) - - if current_text: - blocks.append({"type": current_type, "text": " ".join(current_text)}) - - return blocks +""" +Main parser module orchestrating the RAG pipeline. +Combines PDF extraction, NLP processing, and SQL data generation. +""" +from typing import Generator, Dict, Any, Optional, List +from extractors import parse_hall_text +from nlp import filter_text, split_into_blocks, extract_keywords +from generators import generate_sql def prepare_text_for_rag( - pdf_path, model, chapter_name="unknown", start_page=1, end_page=None -): - """ - Полный pipeline для RAG: парсинг → фильтрация → блоки → эмбеддинги. + pdf_path: str, + model: Any, + chapter_name: str = "unknown", + start_page: int = 1, + end_page: Optional[int] = None, +) -> Generator[Dict[str, Any], None, None]: + """Complete RAG pipeline: extraction → cleaning → blocking → embeddings. + + Processes PDF document through complete pipeline including text extraction, + cleaning, block identification, keyword extraction, and embedding generation + for both individual terms and full text blocks. + + Args: + pdf_path: Path to the PDF file. + model: Sentence Transformer model for encoding text and terms. + chapter_name: Identifier for the chapter/document. Defaults to 'unknown'. + start_page: Starting page number (1-indexed). Defaults to 1. + end_page: Ending page number (inclusive). If None, processes all pages. + + Yields: + Dictionary with keys: + - chapter_name (str): Document identifier. + - terms (List[str]): Original term forms from text. + - definitions (List[str]): Context/definition text for each term. + - chunk_text (str): Full text block. + - terms_embeddings (List[List[float]]): Embeddings of normalized terms. + - chunk_embedding (List[float]): Embedding of full chunk. """ chunks = parse_hall_text(pdf_path, start_page, end_page) @@ -124,94 +51,33 @@ def prepare_text_for_rag( if not block_text.strip(): continue - keywords = extract_keywords(block_text) - embedding = model.encode(block_text).tolist() + # Extract keywords (original and normalized forms) + keyword_pairs = extract_keywords(block_text) + + # Separate original and normalized terms + terms_original: List[str] = [orig for orig, norm in keyword_pairs] + terms_normalized: List[str] = [norm for orig, norm in keyword_pairs] + + # Generate embeddings for each normalized term + terms_embeddings: List[List[float]] = [] + for norm_term in terms_normalized: + term_embedding = model.encode(norm_term).tolist() + terms_embeddings.append(term_embedding) + + # Generate embedding for entire chunk + chunk_embedding: List[float] = model.encode(block_text).tolist() + + # Use full block text as definition context for each term + definitions: List[str] = [block_text] * len(terms_original) yield { "chapter_name": chapter_name, - "keywords": keywords, + "terms": terms_original, + "definitions": definitions, "chunk_text": block_text, - "embedding": embedding, + "terms_embeddings": terms_embeddings, + "chunk_embedding": chunk_embedding, } -def escape_sql_string(s): - """Помощник для эскейпинга одинарных кавычек в SQL.""" - return s.replace("'", "''") - - -INSERT_VALUES_PER_STATEMENT = 50 - - -def format_insert_values_row(item): - """Одна строка для списка VALUES (...), (...), ...""" - kw_cleaned = [ - str(kw).replace("'", "").replace('"', "") for kw in item["keywords"][:10] - ] - keywords_pg = "{" + ",".join(kw_cleaned) + "}" - embedding_str = "[" + ",".join(map(str, item["embedding"])) + "]" - return ( - f"('{escape_sql_string(item['chapter_name'])}', " - f"'{keywords_pg}', " - f"'{escape_sql_string(item['chunk_text'])}', " - f"'{embedding_str}')" - ) - - -def generate_sql(data_generator, output_file="init.sql"): - """ - Генерирует init.sql: DDL и пакетные INSERT (не по одной строке на команду). - """ - init_script = [ - "-- Инициализация таблиц для RAG", - "CREATE EXTENSION IF NOT EXISTS vector;", - "CREATE TABLE IF NOT EXISTS documents (", - " id SERIAL PRIMARY KEY,", - " chapter_name TEXT,", - " keywords TEXT[],", - " chunk_text TEXT,", - " embedding vector(384)", - ");", - "", - f"-- Данные: один INSERT содержит до {INSERT_VALUES_PER_STATEMENT} строк.", - "", - ] - - insert_header = ( - "INSERT INTO documents (chapter_name, keywords, chunk_text, embedding) VALUES\n" - ) - - with open(output_file, "w", encoding="utf-8") as f: - for line in init_script: - f.write(line + "\n") - - count = 0 - batch = [] - statements = 0 - - def flush_batch(): - nonlocal batch, statements - if not batch: - return - f.write(insert_header) - f.write(",\n".join(batch)) - f.write(";\n\n") - statements += 1 - batch = [] - - for item in data_generator: - count += 1 - batch.append(format_insert_values_row(item)) - - if len(batch) >= INSERT_VALUES_PER_STATEMENT: - flush_batch() - - if count % 10 == 0: - print(f"✍️ Обработано и записано {count} логических блоков в SQL...") - - flush_batch() - - print( - f"✅ SQL скрипт успешно сгенерирован: {output_file}. " - f"Всего записей (блоков): {count}, операторов INSERT: {statements}" - ) +__all__ = ["prepare_text_for_rag", "generate_sql"] diff --git a/parser/requirements.txt b/parser/requirements.txt index c4165eb..2bc9892 100644 --- a/parser/requirements.txt +++ b/parser/requirements.txt @@ -3,3 +3,4 @@ sentence-transformers onnxruntime ruff yake +psycopg2-binary From ffd2c953af89622017f83dcd805fbb666e28dc39 Mon Sep 17 00:00:00 2001 From: andreikurakin Date: Thu, 9 Apr 2026 20:25:25 +0300 Subject: [PATCH 4/5] fix linker errors --- parser/generators/sql_generator.py | 14 +++++++++----- parser/nlp/keyword_extractor.py | 4 ++-- parser/nlp/text_processor.py | 19 ++++++++++++++----- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/parser/generators/sql_generator.py b/parser/generators/sql_generator.py index c0caebe..85ecb9c 100644 --- a/parser/generators/sql_generator.py +++ b/parser/generators/sql_generator.py @@ -33,8 +33,8 @@ def format_insert_values_rows(item: Dict[str, Any]) -> List[str]: rows = [] # Iterate through terms and create a card for each - for i, (term, definition, embedding) in enumerate( - zip(item["terms"], item["definitions"], item["terms_embeddings"]) + for term, definition, embedding in zip( + item["terms"], item["definitions"], item["terms_embeddings"], strict=False ): # Format embedding as PostgreSQL vector embedding_str = "[" + ",".join(map(str, embedding)) + "]" @@ -92,7 +92,9 @@ def generate_sql( "-- Tracks spaced repetition metrics for each card", "CREATE TABLE IF NOT EXISTS card_progress (", " id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),", - " card_id UUID NOT NULL UNIQUE REFERENCES card_contents (id) ON DELETE CASCADE,", + " card_id UUID NOT NULL UNIQUE", + " REFERENCES card_contents (id)", + " ON DELETE CASCADE,", " interval BIGINT NOT NULL DEFAULT 0,", " easiness REAL NOT NULL DEFAULT 2.5,", " repetitions INT NOT NULL DEFAULT 0,", @@ -100,14 +102,16 @@ def generate_sql( " updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()", ");", "", - "CREATE INDEX IF NOT EXISTS idx_progress_next_review ON card_progress (next_review_at);", + "CREATE INDEX IF NOT EXISTS idx_progress_next_review", + "ON card_progress (next_review_at);", "", f"-- Data: each INSERT contains up to {INSERT_VALUES_PER_STATEMENT} rows.", "", ] insert_header = ( - "INSERT INTO card_contents (front_text, back_text, embedding, source_info) VALUES\n" + "INSERT INTO card_contents " + "(front_text, back_text, embedding, source_info) VALUES\n" ) with open(output_file, "w", encoding="utf-8") as f: diff --git a/parser/nlp/keyword_extractor.py b/parser/nlp/keyword_extractor.py index 74f30b5..a578883 100644 --- a/parser/nlp/keyword_extractor.py +++ b/parser/nlp/keyword_extractor.py @@ -11,7 +11,7 @@ 'выше', 'ниже', 'раньше', 'позже', 'всегда', 'никогда', 'часто', 'редко', 'который', 'какой', 'какая', 'какое', 'кто', 'где', 'когда', 'почему', 'зачем', 'я', 'ты', 'он', 'она', 'оно', 'мы', 'вы', 'они', 'ме', 'те', 'же', 'ее', 'его', - 'более', 'менее', 'очень', 'же', 'же', + 'более', 'менее', 'очень', } @@ -46,7 +46,7 @@ def extract_keywords(text: str, top_n: int = 20) -> List[Tuple[str, str]]: filtered_keywords: List[Tuple[str, str]] = [] seen_normalized: Set[str] = set() - for kw, score in keywords_raw: + for kw, _score in keywords_raw: kw_original = kw.strip() # Preserve original word form kw_normalized = kw_original.lower() # Normalize for filtering diff --git a/parser/nlp/text_processor.py b/parser/nlp/text_processor.py index acf83fd..c331e2f 100644 --- a/parser/nlp/text_processor.py +++ b/parser/nlp/text_processor.py @@ -39,13 +39,22 @@ def filter_text(text: str) -> str: return text.strip() -def split_into_blocks(text): - """ - Разбивает текст на логические блоки (Определения, Теоремы и обычный текст). +def split_into_blocks(text: str) -> List[Dict[str, str]]: + """Split text into logical blocks by type. + + Identifies and separates text into blocks marked as definitions, theorems, + or general content based on Russian language keywords. + + Args: + text: Cleaned text to split into blocks. + + Returns: + List of dictionaries with 'type' and 'text' keys. Types are: + 'definition', 'theorem', or 'general'. """ - blocks = [] + blocks: List[Dict[str, str]] = [] current_type = "general" - current_text = [] + current_text: List[str] = [] sentences = re.split(r"(?<=[.!?])\s+", text) From 220c3b97937ad8376ed9d0dbba9a1d312b349f78 Mon Sep 17 00:00:00 2001 From: andreikurakin Date: Thu, 9 Apr 2026 23:56:43 +0300 Subject: [PATCH 5/5] linter --- parser/extractors/__init__.py | 1 + parser/extractors/pdf_extractor.py | 7 +- parser/generators/__init__.py | 1 + parser/generators/sql_generator.py | 31 +++++---- parser/main.py | 1 + parser/nlp/__init__.py | 1 + parser/nlp/keyword_extractor.py | 102 ++++++++++++++++++++++------- parser/nlp/text_processor.py | 14 ++-- parser/parser.py | 15 +++-- 9 files changed, 118 insertions(+), 55 deletions(-) diff --git a/parser/extractors/__init__.py b/parser/extractors/__init__.py index 5cc1457..e409a0b 100644 --- a/parser/extractors/__init__.py +++ b/parser/extractors/__init__.py @@ -1,4 +1,5 @@ """PDF extraction and text processing module.""" + from .pdf_extractor import parse_hall_text __all__ = ["parse_hall_text"] diff --git a/parser/extractors/pdf_extractor.py b/parser/extractors/pdf_extractor.py index 5fbde83..0ffbc95 100644 --- a/parser/extractors/pdf_extractor.py +++ b/parser/extractors/pdf_extractor.py @@ -1,4 +1,5 @@ """PDF text extraction module.""" + from typing import Generator, Dict, Optional import pdfplumber @@ -7,16 +8,16 @@ def parse_hall_text( pdf_path: str, start_page: int = 1, end_page: Optional[int] = None ) -> Generator[Dict[str, object], None, None]: """Extract text from PDF using multiple fallback methods. - + Attempts to extract text from each page using pdfplumber. If initial extraction fails or produces insufficient text, falls back to adjusted tolerance settings and table extraction. - + Args: pdf_path: Path to the PDF file. start_page: Starting page number (1-indexed). Defaults to 1. end_page: Ending page number (inclusive). If None, processes all pages. - + Yields: Dictionary containing 'page' (int) and 'text' (str) keys. Only yields pages with extracted text length > 10 characters. diff --git a/parser/generators/__init__.py b/parser/generators/__init__.py index 3762352..38ce22c 100644 --- a/parser/generators/__init__.py +++ b/parser/generators/__init__.py @@ -1,4 +1,5 @@ """SQL generation and data mapping module.""" + from .sql_generator import generate_sql, escape_sql_string, format_insert_values_rows __all__ = ["generate_sql", "escape_sql_string", "format_insert_values_rows"] diff --git a/parser/generators/sql_generator.py b/parser/generators/sql_generator.py index 85ecb9c..439c414 100644 --- a/parser/generators/sql_generator.py +++ b/parser/generators/sql_generator.py @@ -1,4 +1,5 @@ """SQL generation and data formatting for database initialization.""" + from typing import Dict, Any, List INSERT_VALUES_PER_STATEMENT = 50 @@ -6,10 +7,10 @@ def escape_sql_string(s: str) -> str: """Escape single quotes in SQL string values. - + Args: s: String to escape. - + Returns: String with single quotes escaped for SQL. """ @@ -18,30 +19,30 @@ def escape_sql_string(s: str) -> str: def format_insert_values_rows(item: Dict[str, Any]) -> List[str]: """Format data items as multiple SQL VALUES rows (one per term). - + Converts extracted terms and their embeddings into individual card_contents records. Each term becomes a separate flashcard with its definition and embedding. - + Args: item: Dictionary with keys: chapter_name, terms, definitions, chunk_text, terms_embeddings. - + Returns: List of SQL VALUES rows, one per extracted term. """ rows = [] - + # Iterate through terms and create a card for each for term, definition, embedding in zip( item["terms"], item["definitions"], item["terms_embeddings"], strict=False ): # Format embedding as PostgreSQL vector embedding_str = "[" + ",".join(map(str, embedding)) + "]" - + # Create source info: chapter + chunk context source_info = f"{item['chapter_name']}|{item['chunk_text'][:200]}" - + row = ( f"('{escape_sql_string(term)}', " f"'{escape_sql_string(definition[:500])}', " @@ -49,29 +50,27 @@ def format_insert_values_rows(item: Dict[str, Any]) -> List[str]: f"'{escape_sql_string(source_info)}')" ) rows.append(row) - + return rows -def generate_sql( - data_generator: Any, output_file: str = "init.sql" -) -> None: +def generate_sql(data_generator: Any, output_file: str = "init.sql") -> None: """Generate SQL initialization script with DDL and batch INSERT statements. - + Creates a PostgreSQL initialization script containing table creation (with vector extension) and batch INSERT statements for card_contents and card_progress tables. Batches are sized for optimal performance. - + Args: data_generator: Generator/iterable yielding data dictionaries. output_file: Path where SQL script will be written. Defaults to 'init.sql'. - + Returns: None. Writes directly to output_file. """ init_script = [ "-- RAG Flashcard Schema", - "CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";", + 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";', "CREATE EXTENSION IF NOT EXISTS vector;", "", "-- Table: card_contents", diff --git a/parser/main.py b/parser/main.py index 53e3e7c..6711803 100644 --- a/parser/main.py +++ b/parser/main.py @@ -1,4 +1,5 @@ """Entry point for RAG document processing pipeline.""" + from pathlib import Path from typing import Generator, Dict, Any from sentence_transformers import SentenceTransformer diff --git a/parser/nlp/__init__.py b/parser/nlp/__init__.py index 2553d95..53de02c 100644 --- a/parser/nlp/__init__.py +++ b/parser/nlp/__init__.py @@ -1,4 +1,5 @@ """NLP processing module for text cleaning and keyword extraction.""" + from .text_processor import filter_text, split_into_blocks from .keyword_extractor import extract_keywords diff --git a/parser/nlp/keyword_extractor.py b/parser/nlp/keyword_extractor.py index a578883..ab2a12f 100644 --- a/parser/nlp/keyword_extractor.py +++ b/parser/nlp/keyword_extractor.py @@ -1,83 +1,141 @@ """Keyword extraction using YAKE algorithm.""" + from typing import List, Tuple, Set import string import yake # Russian stop words for filtering RU_STOP_WORDS: Set[str] = { - 'что', 'это', 'как', 'за', 'из', 'на', 'при', 'в', 'с', 'по', 'к', 'до', - 'и', 'или', 'но', 'а', 'не', 'ни', 'да', 'нет', 'если', 'то', - 'для', 'от', 'об', 'о', 'без', 'через', 'между', 'перед', 'после', - 'выше', 'ниже', 'раньше', 'позже', 'всегда', 'никогда', 'часто', 'редко', - 'который', 'какой', 'какая', 'какое', 'кто', 'где', 'когда', 'почему', 'зачем', - 'я', 'ты', 'он', 'она', 'оно', 'мы', 'вы', 'они', 'ме', 'те', 'же', 'ее', 'его', - 'более', 'менее', 'очень', + "что", + "это", + "как", + "за", + "из", + "на", + "при", + "в", + "с", + "по", + "к", + "до", + "и", + "или", + "но", + "а", + "не", + "ни", + "да", + "нет", + "если", + "то", + "для", + "от", + "об", + "о", + "без", + "через", + "между", + "перед", + "после", + "выше", + "ниже", + "раньше", + "позже", + "всегда", + "никогда", + "часто", + "редко", + "который", + "какой", + "какая", + "какое", + "кто", + "где", + "когда", + "почему", + "зачем", + "я", + "ты", + "он", + "она", + "оно", + "мы", + "вы", + "они", + "ме", + "те", + "же", + "ее", + "его", + "более", + "менее", + "очень", } def extract_keywords(text: str, top_n: int = 20) -> List[Tuple[str, str]]: """Extract and filter keywords/terms from text. - + Uses YAKE algorithm to identify single and two-word terms, then filters based on Russian stop words, minimum length, and stop word ratio. Original word forms are preserved while normalized versions are used for embedding generation. - + Args: text: Input text to extract keywords from. top_n: Maximum number of keywords to return. Defaults to 20. - + Returns: List of tuples (original_word, normalized_word) for each extracted keyword. Normalized words are lowercase for consistent embedding generation. """ if not text.strip(): return [] - + # Extract single and two-word terms using YAKE try: kw_extractor = yake.KeywordExtractor( - lan="ru", n=2, dedupLim=0.9, top=top_n*2, features=None + lan="ru", n=2, dedupLim=0.9, top=top_n * 2, features=None ) keywords_raw = kw_extractor.extract_keywords(text) except Exception: return [] - + filtered_keywords: List[Tuple[str, str]] = [] seen_normalized: Set[str] = set() - + for kw, _score in keywords_raw: kw_original = kw.strip() # Preserve original word form kw_normalized = kw_original.lower() # Normalize for filtering - + # Skip if empty or exact match to stop word if not kw_normalized or kw_normalized in RU_STOP_WORDS: continue - + # Check minimum length (at least 2 characters) if len(kw_normalized) < 2: continue - + # Remove words containing only punctuation or digits if all(c in string.punctuation + string.digits for c in kw_normalized): continue - + # Remove single-word terms that are stop words words = kw_normalized.split() if all(w in RU_STOP_WORDS for w in words): continue - + # Remove phrases where more than half are stop words stop_word_ratio = sum(1 for w in words if w in RU_STOP_WORDS) / len(words) if stop_word_ratio > 0.5: continue - + # Remove duplicates by normalized form if kw_normalized not in seen_normalized: filtered_keywords.append((kw_original, kw_normalized)) seen_normalized.add(kw_normalized) - + if len(filtered_keywords) >= top_n: break - + return filtered_keywords diff --git a/parser/nlp/text_processor.py b/parser/nlp/text_processor.py index c331e2f..537b026 100644 --- a/parser/nlp/text_processor.py +++ b/parser/nlp/text_processor.py @@ -1,18 +1,18 @@ - """Text cleaning and block splitting module.""" + from typing import List, Dict import re def filter_text(text: str) -> str: """Remove noise and service information from text. - + Removes algorithm blocks, figure/table references, page numbers, URLs, email addresses, excess whitespace, and hanging punctuation. - + Args: text: Raw text to clean. - + Returns: Cleaned text with all filtered elements removed. """ @@ -41,13 +41,13 @@ def filter_text(text: str) -> str: def split_into_blocks(text: str) -> List[Dict[str, str]]: """Split text into logical blocks by type. - + Identifies and separates text into blocks marked as definitions, theorems, or general content based on Russian language keywords. - + Args: text: Cleaned text to split into blocks. - + Returns: List of dictionaries with 'type' and 'text' keys. Types are: 'definition', 'theorem', or 'general'. diff --git a/parser/parser.py b/parser/parser.py index deeebb2..2ad754b 100644 --- a/parser/parser.py +++ b/parser/parser.py @@ -2,6 +2,7 @@ Main parser module orchestrating the RAG pipeline. Combines PDF extraction, NLP processing, and SQL data generation. """ + from typing import Generator, Dict, Any, Optional, List from extractors import parse_hall_text from nlp import filter_text, split_into_blocks, extract_keywords @@ -16,18 +17,18 @@ def prepare_text_for_rag( end_page: Optional[int] = None, ) -> Generator[Dict[str, Any], None, None]: """Complete RAG pipeline: extraction → cleaning → blocking → embeddings. - + Processes PDF document through complete pipeline including text extraction, cleaning, block identification, keyword extraction, and embedding generation for both individual terms and full text blocks. - + Args: pdf_path: Path to the PDF file. model: Sentence Transformer model for encoding text and terms. chapter_name: Identifier for the chapter/document. Defaults to 'unknown'. start_page: Starting page number (1-indexed). Defaults to 1. end_page: Ending page number (inclusive). If None, processes all pages. - + Yields: Dictionary with keys: - chapter_name (str): Document identifier. @@ -53,20 +54,20 @@ def prepare_text_for_rag( # Extract keywords (original and normalized forms) keyword_pairs = extract_keywords(block_text) - + # Separate original and normalized terms terms_original: List[str] = [orig for orig, norm in keyword_pairs] terms_normalized: List[str] = [norm for orig, norm in keyword_pairs] - + # Generate embeddings for each normalized term terms_embeddings: List[List[float]] = [] for norm_term in terms_normalized: term_embedding = model.encode(norm_term).tolist() terms_embeddings.append(term_embedding) - + # Generate embedding for entire chunk chunk_embedding: List[float] = model.encode(block_text).tolist() - + # Use full block text as definition context for each term definitions: List[str] = [block_text] * len(terms_original)