diff --git a/parser/extractors/__init__.py b/parser/extractors/__init__.py new file mode 100644 index 0000000..e409a0b --- /dev/null +++ b/parser/extractors/__init__.py @@ -0,0 +1,5 @@ +"""PDF extraction and text processing module.""" + +from .pdf_extractor import parse_hall_text + +__all__ = ["parse_hall_text"] diff --git a/parser/extractors/pdf_extractor.py b/parser/extractors/pdf_extractor.py new file mode 100644 index 0000000..0ffbc95 --- /dev/null +++ b/parser/extractors/pdf_extractor.py @@ -0,0 +1,62 @@ +"""PDF text extraction module.""" + +from typing import Generator, Dict, Optional +import pdfplumber + + +def parse_hall_text( + pdf_path: str, start_page: int = 1, end_page: Optional[int] = None +) -> Generator[Dict[str, object], None, None]: + """Extract text from PDF using multiple fallback methods. + + Attempts to extract text from each page using pdfplumber. If initial extraction + fails or produces insufficient text, falls back to adjusted tolerance settings + and table extraction. + + Args: + pdf_path: Path to the PDF file. + start_page: Starting page number (1-indexed). Defaults to 1. + end_page: Ending page number (inclusive). If None, processes all pages. + + Yields: + Dictionary containing 'page' (int) and 'text' (str) keys. + Only yields pages with extracted text length > 10 characters. + """ + with pdfplumber.open(pdf_path) as pdf: + for page_num, page in enumerate(pdf.pages, 1): + if page_num < start_page: + continue + if end_page and page_num > end_page: + break + + text = page.extract_text() + + if not text or len(text.strip()) < 10: + try: + text = page.extract_text(x_tolerance=2, y_tolerance=2) + except Exception: + pass + + if not text or len(text.strip()) < 10: + tables = page.extract_tables() + if tables: + text = "\n".join( + [ + "\n".join([str(cell) for cell in row]) + for table in tables + for row in table + ] + ) + + if text and len(text.strip()) > 10: + yield { + "page": page_num, + "text": text, + } + else: + print(f"Warning: Page {page_num} - no text extracted") + + page.flush_cache() + + if page_num % 10 == 0: + print(f"Processed {page_num} pages...") diff --git a/parser/generators/__init__.py b/parser/generators/__init__.py new file mode 100644 index 0000000..38ce22c --- /dev/null +++ b/parser/generators/__init__.py @@ -0,0 +1,5 @@ +"""SQL generation and data mapping module.""" + +from .sql_generator import generate_sql, escape_sql_string, format_insert_values_rows + +__all__ = ["generate_sql", "escape_sql_string", "format_insert_values_rows"] diff --git a/parser/generators/sql_generator.py b/parser/generators/sql_generator.py new file mode 100644 index 0000000..439c414 --- /dev/null +++ b/parser/generators/sql_generator.py @@ -0,0 +1,152 @@ +"""SQL generation and data formatting for database initialization.""" + +from typing import Dict, Any, List + +INSERT_VALUES_PER_STATEMENT = 50 + + +def escape_sql_string(s: str) -> str: + """Escape single quotes in SQL string values. + + Args: + s: String to escape. + + Returns: + String with single quotes escaped for SQL. + """ + return s.replace("'", "''") + + +def format_insert_values_rows(item: Dict[str, Any]) -> List[str]: + """Format data items as multiple SQL VALUES rows (one per term). + + Converts extracted terms and their embeddings into individual card_contents + records. Each term becomes a separate flashcard with its definition and + embedding. + + Args: + item: Dictionary with keys: chapter_name, terms, definitions, + chunk_text, terms_embeddings. + + Returns: + List of SQL VALUES rows, one per extracted term. + """ + rows = [] + + # Iterate through terms and create a card for each + for term, definition, embedding in zip( + item["terms"], item["definitions"], item["terms_embeddings"], strict=False + ): + # Format embedding as PostgreSQL vector + embedding_str = "[" + ",".join(map(str, embedding)) + "]" + + # Create source info: chapter + chunk context + source_info = f"{item['chapter_name']}|{item['chunk_text'][:200]}" + + row = ( + f"('{escape_sql_string(term)}', " + f"'{escape_sql_string(definition[:500])}', " + f"'{embedding_str}', " + f"'{escape_sql_string(source_info)}')" + ) + rows.append(row) + + return rows + + +def generate_sql(data_generator: Any, output_file: str = "init.sql") -> None: + """Generate SQL initialization script with DDL and batch INSERT statements. + + Creates a PostgreSQL initialization script containing table creation + (with vector extension) and batch INSERT statements for card_contents and + card_progress tables. Batches are sized for optimal performance. + + Args: + data_generator: Generator/iterable yielding data dictionaries. + output_file: Path where SQL script will be written. Defaults to 'init.sql'. + + Returns: + None. Writes directly to output_file. + """ + init_script = [ + "-- RAG Flashcard Schema", + 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";', + "CREATE EXTENSION IF NOT EXISTS vector;", + "", + "-- Table: card_contents", + "-- Stores individual terms with embeddings and source context", + "CREATE TABLE IF NOT EXISTS card_contents (", + " id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),", + " front_text TEXT NOT NULL,", + " back_text TEXT NOT NULL,", + " embedding vector(384),", + " source_info TEXT,", + " created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()", + ");", + "", + "-- HNSW index for similarity search", + "CREATE INDEX ON card_contents USING hnsw (embedding vector_cosine_ops);", + "", + "-- Table: card_progress", + "-- Tracks spaced repetition metrics for each card", + "CREATE TABLE IF NOT EXISTS card_progress (", + " id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),", + " card_id UUID NOT NULL UNIQUE", + " REFERENCES card_contents (id)", + " ON DELETE CASCADE,", + " interval BIGINT NOT NULL DEFAULT 0,", + " easiness REAL NOT NULL DEFAULT 2.5,", + " repetitions INT NOT NULL DEFAULT 0,", + " next_review_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),", + " updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()", + ");", + "", + "CREATE INDEX IF NOT EXISTS idx_progress_next_review", + "ON card_progress (next_review_at);", + "", + f"-- Data: each INSERT contains up to {INSERT_VALUES_PER_STATEMENT} rows.", + "", + ] + + insert_header = ( + "INSERT INTO card_contents " + "(front_text, back_text, embedding, source_info) VALUES\n" + ) + + with open(output_file, "w", encoding="utf-8") as f: + for line in init_script: + f.write(line + "\n") + + count = 0 + batch = [] + statements = 0 + + def flush_batch(): + nonlocal batch, statements + if not batch: + return + f.write(insert_header) + f.write(",\n".join(batch)) + f.write(";\n\n") + statements += 1 + batch = [] + + for item in data_generator: + # Each item can produce multiple rows (one per term) + rows = format_insert_values_rows(item) + for row in rows: + count += 1 + batch.append(row) + + if len(batch) >= INSERT_VALUES_PER_STATEMENT: + flush_batch() + + if count % 100 == 0: + print(f"Processed and written {count} flashcards to SQL...") + + flush_batch() + + print( + f"SQL script generated successfully: {output_file}. " + f"Total flashcards: {count}, INSERT statements: {statements}" + ) diff --git a/parser/main.py b/parser/main.py new file mode 100644 index 0000000..6711803 --- /dev/null +++ b/parser/main.py @@ -0,0 +1,56 @@ +"""Entry point for RAG document processing pipeline.""" + +from pathlib import Path +from typing import Generator, Dict, Any +from sentence_transformers import SentenceTransformer +from parser import prepare_text_for_rag +from generators import generate_sql + + +def main() -> None: + """Initialize model and execute RAG pipeline on all PDF files in data directory.""" + print( + "Initializing model " + "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2..." + ) + # Cache models locally in /parser/models/ + models_dir = Path(__file__).parent / "models" + models_dir.mkdir(parents=True, exist_ok=True) + try: + model = SentenceTransformer( + "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + cache_folder=str(models_dir), + ) + except Exception as e: + print(f"Error loading model: {e}") + return + + data_dir = Path(__file__).parent.parent / "data" + if not data_dir.exists(): + print( + f"Data directory {data_dir} not found. " + "Creating empty directory (add PDF files to it)." + ) + data_dir.mkdir(parents=True, exist_ok=True) + + pdf_files = list(data_dir.glob("*.pdf")) + if not pdf_files: + print(f"No PDF files found in {data_dir}.") + return + + print(f"Found {len(pdf_files)} PDF files to process.") + + def all_pdfs_generator() -> Generator[Dict[str, Any], None, None]: + """Generate processed data from all PDF files.""" + for pdf_path in pdf_files: + print(f"Processing document: {pdf_path.name}") + # Extract chapter name from filename + chapter_name = pdf_path.stem + yield from prepare_text_for_rag(pdf_path, model, chapter_name=chapter_name) + + output_sql = Path(__file__).parent / "init.sql" + generate_sql(all_pdfs_generator(), output_file=str(output_sql)) + + +if __name__ == "__main__": + main() diff --git a/parser/nlp/__init__.py b/parser/nlp/__init__.py new file mode 100644 index 0000000..53de02c --- /dev/null +++ b/parser/nlp/__init__.py @@ -0,0 +1,6 @@ +"""NLP processing module for text cleaning and keyword extraction.""" + +from .text_processor import filter_text, split_into_blocks +from .keyword_extractor import extract_keywords + +__all__ = ["filter_text", "split_into_blocks", "extract_keywords"] diff --git a/parser/nlp/keyword_extractor.py b/parser/nlp/keyword_extractor.py new file mode 100644 index 0000000..ab2a12f --- /dev/null +++ b/parser/nlp/keyword_extractor.py @@ -0,0 +1,141 @@ +"""Keyword extraction using YAKE algorithm.""" + +from typing import List, Tuple, Set +import string +import yake + +# Russian stop words for filtering +RU_STOP_WORDS: Set[str] = { + "что", + "это", + "как", + "за", + "из", + "на", + "при", + "в", + "с", + "по", + "к", + "до", + "и", + "или", + "но", + "а", + "не", + "ни", + "да", + "нет", + "если", + "то", + "для", + "от", + "об", + "о", + "без", + "через", + "между", + "перед", + "после", + "выше", + "ниже", + "раньше", + "позже", + "всегда", + "никогда", + "часто", + "редко", + "который", + "какой", + "какая", + "какое", + "кто", + "где", + "когда", + "почему", + "зачем", + "я", + "ты", + "он", + "она", + "оно", + "мы", + "вы", + "они", + "ме", + "те", + "же", + "ее", + "его", + "более", + "менее", + "очень", +} + + +def extract_keywords(text: str, top_n: int = 20) -> List[Tuple[str, str]]: + """Extract and filter keywords/terms from text. + + Uses YAKE algorithm to identify single and two-word terms, then filters + based on Russian stop words, minimum length, and stop word ratio. + Original word forms are preserved while normalized versions are used for + embedding generation. + + Args: + text: Input text to extract keywords from. + top_n: Maximum number of keywords to return. Defaults to 20. + + Returns: + List of tuples (original_word, normalized_word) for each extracted keyword. + Normalized words are lowercase for consistent embedding generation. + """ + if not text.strip(): + return [] + + # Extract single and two-word terms using YAKE + try: + kw_extractor = yake.KeywordExtractor( + lan="ru", n=2, dedupLim=0.9, top=top_n * 2, features=None + ) + keywords_raw = kw_extractor.extract_keywords(text) + except Exception: + return [] + + filtered_keywords: List[Tuple[str, str]] = [] + seen_normalized: Set[str] = set() + + for kw, _score in keywords_raw: + kw_original = kw.strip() # Preserve original word form + kw_normalized = kw_original.lower() # Normalize for filtering + + # Skip if empty or exact match to stop word + if not kw_normalized or kw_normalized in RU_STOP_WORDS: + continue + + # Check minimum length (at least 2 characters) + if len(kw_normalized) < 2: + continue + + # Remove words containing only punctuation or digits + if all(c in string.punctuation + string.digits for c in kw_normalized): + continue + + # Remove single-word terms that are stop words + words = kw_normalized.split() + if all(w in RU_STOP_WORDS for w in words): + continue + + # Remove phrases where more than half are stop words + stop_word_ratio = sum(1 for w in words if w in RU_STOP_WORDS) / len(words) + if stop_word_ratio > 0.5: + continue + + # Remove duplicates by normalized form + if kw_normalized not in seen_normalized: + filtered_keywords.append((kw_original, kw_normalized)) + seen_normalized.add(kw_normalized) + + if len(filtered_keywords) >= top_n: + break + + return filtered_keywords diff --git a/parser/nlp/text_processor.py b/parser/nlp/text_processor.py new file mode 100644 index 0000000..537b026 --- /dev/null +++ b/parser/nlp/text_processor.py @@ -0,0 +1,80 @@ +"""Text cleaning and block splitting module.""" + +from typing import List, Dict +import re + + +def filter_text(text: str) -> str: + """Remove noise and service information from text. + + Removes algorithm blocks, figure/table references, page numbers, URLs, + email addresses, excess whitespace, and hanging punctuation. + + Args: + text: Raw text to clean. + + Returns: + Cleaned text with all filtered elements removed. + """ + # Remove algorithm blocks + text = re.sub( + r"---algorithm---.*?---end---", "", text, flags=re.DOTALL | re.IGNORECASE + ) + # Remove figure and table references + text = re.sub(r"(?:рис|fig)\.\s*\d+[^\n]*", "", text, flags=re.IGNORECASE) + text = re.sub(r"(?:табл|table)\.\s*\d+[^\n]*", "", text, flags=re.IGNORECASE) + # Remove page numbers and service elements + text = re.sub(r"\(стр\.\s+\d+\)", "", text, flags=re.IGNORECASE) + text = re.sub(r"\[страница\s+\d+\]", "", text, flags=re.IGNORECASE) + text = re.sub(r"^-+\s*\d+\s*-+$", "", text, flags=re.MULTILINE) + # Remove URLs + text = re.sub(r"https?://\S+", "", text) + # Remove email addresses + text = re.sub(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", "", text) + # Remove excess whitespace and line breaks + text = re.sub(r"\n+", "\n", text) + text = re.sub(r"\s+", " ", text) + # Remove hanging punctuation + text = re.sub(r"^\s*[^\w\s]*\s+", "", text, flags=re.MULTILINE) + return text.strip() + + +def split_into_blocks(text: str) -> List[Dict[str, str]]: + """Split text into logical blocks by type. + + Identifies and separates text into blocks marked as definitions, theorems, + or general content based on Russian language keywords. + + Args: + text: Cleaned text to split into blocks. + + Returns: + List of dictionaries with 'type' and 'text' keys. Types are: + 'definition', 'theorem', or 'general'. + """ + blocks: List[Dict[str, str]] = [] + current_type = "general" + current_text: List[str] = [] + + sentences = re.split(r"(?<=[.!?])\s+", text) + + for sentence in sentences: + lower_s = sentence.lower() + + if lower_s.startswith("определение"): + if current_text: + blocks.append({"type": current_type, "text": " ".join(current_text)}) + current_type = "definition" + current_text = [sentence] + elif lower_s.startswith("теорема") or lower_s.startswith("лемма"): + if current_text: + blocks.append({"type": current_type, "text": " ".join(current_text)}) + current_type = "theorem" + current_text = [sentence] + else: + current_text.append(sentence) + + if current_text: + blocks.append({"type": current_type, "text": " ".join(current_text)}) + + return blocks diff --git a/parser/parser.py b/parser/parser.py new file mode 100644 index 0000000..2ad754b --- /dev/null +++ b/parser/parser.py @@ -0,0 +1,84 @@ +""" +Main parser module orchestrating the RAG pipeline. +Combines PDF extraction, NLP processing, and SQL data generation. +""" + +from typing import Generator, Dict, Any, Optional, List +from extractors import parse_hall_text +from nlp import filter_text, split_into_blocks, extract_keywords +from generators import generate_sql + + +def prepare_text_for_rag( + pdf_path: str, + model: Any, + chapter_name: str = "unknown", + start_page: int = 1, + end_page: Optional[int] = None, +) -> Generator[Dict[str, Any], None, None]: + """Complete RAG pipeline: extraction → cleaning → blocking → embeddings. + + Processes PDF document through complete pipeline including text extraction, + cleaning, block identification, keyword extraction, and embedding generation + for both individual terms and full text blocks. + + Args: + pdf_path: Path to the PDF file. + model: Sentence Transformer model for encoding text and terms. + chapter_name: Identifier for the chapter/document. Defaults to 'unknown'. + start_page: Starting page number (1-indexed). Defaults to 1. + end_page: Ending page number (inclusive). If None, processes all pages. + + Yields: + Dictionary with keys: + - chapter_name (str): Document identifier. + - terms (List[str]): Original term forms from text. + - definitions (List[str]): Context/definition text for each term. + - chunk_text (str): Full text block. + - terms_embeddings (List[List[float]]): Embeddings of normalized terms. + - chunk_embedding (List[float]): Embedding of full chunk. + """ + chunks = parse_hall_text(pdf_path, start_page, end_page) + + for chunk in chunks: + cleaned = filter_text(chunk["text"]) + if not cleaned.strip(): + continue + + blocks = split_into_blocks(cleaned) + + for block in blocks: + block_text = block["text"] + if not block_text.strip(): + continue + + # Extract keywords (original and normalized forms) + keyword_pairs = extract_keywords(block_text) + + # Separate original and normalized terms + terms_original: List[str] = [orig for orig, norm in keyword_pairs] + terms_normalized: List[str] = [norm for orig, norm in keyword_pairs] + + # Generate embeddings for each normalized term + terms_embeddings: List[List[float]] = [] + for norm_term in terms_normalized: + term_embedding = model.encode(norm_term).tolist() + terms_embeddings.append(term_embedding) + + # Generate embedding for entire chunk + chunk_embedding: List[float] = model.encode(block_text).tolist() + + # Use full block text as definition context for each term + definitions: List[str] = [block_text] * len(terms_original) + + yield { + "chapter_name": chapter_name, + "terms": terms_original, + "definitions": definitions, + "chunk_text": block_text, + "terms_embeddings": terms_embeddings, + "chunk_embedding": chunk_embedding, + } + + +__all__ = ["prepare_text_for_rag", "generate_sql"] diff --git a/parser/pyproject.toml b/parser/pyproject.toml new file mode 100644 index 0000000..562c548 --- /dev/null +++ b/parser/pyproject.toml @@ -0,0 +1,11 @@ +[tool.ruff] +line-length = 88 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "F", "B"] +ignore = [] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" diff --git a/parser/requirements.txt b/parser/requirements.txt index feb7b9a..2bc9892 100644 --- a/parser/requirements.txt +++ b/parser/requirements.txt @@ -1,4 +1,6 @@ pdfplumber sentence-transformers onnxruntime -psycopg2-binary \ No newline at end of file +ruff +yake +psycopg2-binary