Feature/parser init

parser/extractors/__init__.py

@@ -0,0 +1,5 @@
+"""PDF extraction and text processing module."""
+
+from .pdf_extractor import parse_hall_text
+
+__all__ = ["parse_hall_text"]

parser/extractors/pdf_extractor.py

@@ -0,0 +1,62 @@
+"""PDF text extraction module."""
+
+from typing import Generator, Dict, Optional
+import pdfplumber
+
+
+def parse_hall_text(
+    pdf_path: str, start_page: int = 1, end_page: Optional[int] = None
+) -> Generator[Dict[str, object], None, None]:
+    """Extract text from PDF using multiple fallback methods.
+
+    Attempts to extract text from each page using pdfplumber. If initial extraction
+    fails or produces insufficient text, falls back to adjusted tolerance settings
+    and table extraction.
+
+    Args:
+        pdf_path: Path to the PDF file.
+        start_page: Starting page number (1-indexed). Defaults to 1.
+        end_page: Ending page number (inclusive). If None, processes all pages.
+
+    Yields:
+        Dictionary containing 'page' (int) and 'text' (str) keys.
+        Only yields pages with extracted text length > 10 characters.
+    """
+    with pdfplumber.open(pdf_path) as pdf:
+        for page_num, page in enumerate(pdf.pages, 1):
+            if page_num < start_page:
+                continue
+            if end_page and page_num > end_page:
+                break
+
+            text = page.extract_text()
+
+            if not text or len(text.strip()) < 10:
+                try:
+                    text = page.extract_text(x_tolerance=2, y_tolerance=2)
+                except Exception:
+                    pass
+
+            if not text or len(text.strip()) < 10:
+                tables = page.extract_tables()
+                if tables:
+                    text = "\n".join(
+                        [
+                            "\n".join([str(cell) for cell in row])
+                            for table in tables
+                            for row in table
+                        ]
+                    )
+
+            if text and len(text.strip()) > 10:
+                yield {
+                    "page": page_num,
+                    "text": text,
+                }
+            else:
+                print(f"Warning: Page {page_num} - no text extracted")
+
+            page.flush_cache()
+
+            if page_num % 10 == 0:
+                print(f"Processed {page_num} pages...")

parser/generators/__init__.py

@@ -0,0 +1,5 @@
+"""SQL generation and data mapping module."""
+
+from .sql_generator import generate_sql, escape_sql_string, format_insert_values_rows
+
+__all__ = ["generate_sql", "escape_sql_string", "format_insert_values_rows"]

parser/generators/sql_generator.py

@@ -0,0 +1,152 @@
+"""SQL generation and data formatting for database initialization."""
+
+from typing import Dict, Any, List
+
+INSERT_VALUES_PER_STATEMENT = 50
+
+
+def escape_sql_string(s: str) -> str:
+    """Escape single quotes in SQL string values.
+
+    Args:
+        s: String to escape.
+
+    Returns:
+        String with single quotes escaped for SQL.
+    """
+    return s.replace("'", "''")
+
+
+def format_insert_values_rows(item: Dict[str, Any]) -> List[str]:
+    """Format data items as multiple SQL VALUES rows (one per term).
+
+    Converts extracted terms and their embeddings into individual card_contents
+    records. Each term becomes a separate flashcard with its definition and
+    embedding.
+
+    Args:
+        item: Dictionary with keys: chapter_name, terms, definitions,
+              chunk_text, terms_embeddings.
+
+    Returns:
+        List of SQL VALUES rows, one per extracted term.
+    """
+    rows = []
+
+    # Iterate through terms and create a card for each
+    for term, definition, embedding in zip(
+        item["terms"], item["definitions"], item["terms_embeddings"], strict=False
+    ):
+        # Format embedding as PostgreSQL vector
+        embedding_str = "[" + ",".join(map(str, embedding)) + "]"
+
+        # Create source info: chapter + chunk context
+        source_info = f"{item['chapter_name']}|{item['chunk_text'][:200]}"
+
+        row = (
+            f"('{escape_sql_string(term)}', "
+            f"'{escape_sql_string(definition[:500])}', "
+            f"'{embedding_str}', "
+            f"'{escape_sql_string(source_info)}')"
+        )
+        rows.append(row)
+
+    return rows
+
+
+def generate_sql(data_generator: Any, output_file: str = "init.sql") -> None:
+    """Generate SQL initialization script with DDL and batch INSERT statements.
+
+    Creates a PostgreSQL initialization script containing table creation
+    (with vector extension) and batch INSERT statements for card_contents and
+    card_progress tables. Batches are sized for optimal performance.
+
+    Args:
+        data_generator: Generator/iterable yielding data dictionaries.
+        output_file: Path where SQL script will be written. Defaults to 'init.sql'.
+
+    Returns:
+        None. Writes directly to output_file.
+    """
+    init_script = [
+        "-- RAG Flashcard Schema",
+        'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";',
+        "CREATE EXTENSION IF NOT EXISTS vector;",
+        "",
+        "-- Table: card_contents",
+        "-- Stores individual terms with embeddings and source context",
+        "CREATE TABLE IF NOT EXISTS card_contents (",
+        "    id          UUID PRIMARY KEY DEFAULT uuid_generate_v4(),",
+        "    front_text  TEXT NOT NULL,",
+        "    back_text   TEXT NOT NULL,",
+        "    embedding   vector(384),",
+        "    source_info TEXT,",
+        "    created_at  TIMESTAMPTZ NOT NULL DEFAULT NOW()",
+        ");",
+        "",
+        "-- HNSW index for similarity search",
+        "CREATE INDEX ON card_contents USING hnsw (embedding vector_cosine_ops);",
+        "",
+        "-- Table: card_progress",
+        "-- Tracks spaced repetition metrics for each card",
+        "CREATE TABLE IF NOT EXISTS card_progress (",
+        "    id             UUID PRIMARY KEY DEFAULT uuid_generate_v4(),",
+        "    card_id        UUID NOT NULL UNIQUE",
+        "                   REFERENCES card_contents (id)",
+        "                   ON DELETE CASCADE,",
+        "    interval       BIGINT NOT NULL DEFAULT 0,",
+        "    easiness       REAL NOT NULL DEFAULT 2.5,",
+        "    repetitions    INT NOT NULL DEFAULT 0,",
+        "    next_review_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),",
+        "    updated_at     TIMESTAMPTZ NOT NULL DEFAULT NOW()",
+        ");",
+        "",
+        "CREATE INDEX IF NOT EXISTS idx_progress_next_review",
+        "ON card_progress (next_review_at);",
+        "",
+        f"-- Data: each INSERT contains up to {INSERT_VALUES_PER_STATEMENT} rows.",
+        "",
+    ]
+
+    insert_header = (
+        "INSERT INTO card_contents "
+        "(front_text, back_text, embedding, source_info) VALUES\n"
+    )
+
+    with open(output_file, "w", encoding="utf-8") as f:
+        for line in init_script:
+            f.write(line + "\n")
+
+        count = 0
+        batch = []
+        statements = 0
+
+        def flush_batch():
+            nonlocal batch, statements
+            if not batch:
+                return
+            f.write(insert_header)
+            f.write(",\n".join(batch))
+            f.write(";\n\n")
+            statements += 1
+            batch = []
+
+        for item in data_generator:
+            # Each item can produce multiple rows (one per term)
+            rows = format_insert_values_rows(item)
+            for row in rows:
+                count += 1
+                batch.append(row)
+
+                if len(batch) >= INSERT_VALUES_PER_STATEMENT:
+                    flush_batch()
+
+            if count % 100 == 0:
+                print(f"Processed and written {count} flashcards to SQL...")
+
+        flush_batch()
+
+    print(
+        f"SQL script generated successfully: {output_file}. "
+        f"Total flashcards: {count}, INSERT statements: {statements}"
+    )

parser/main.py

@@ -0,0 +1,56 @@
+"""Entry point for RAG document processing pipeline."""
+
+from pathlib import Path
+from typing import Generator, Dict, Any
+from sentence_transformers import SentenceTransformer
+from parser import prepare_text_for_rag
+from generators import generate_sql
+
+
+def main() -> None:
+    """Initialize model and execute RAG pipeline on all PDF files in data directory."""
+    print(
+        "Initializing model "
+        "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2..."
+    )
+    # Cache models locally in /parser/models/
+    models_dir = Path(__file__).parent / "models"
+    models_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        model = SentenceTransformer(
+            "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
+            cache_folder=str(models_dir),
+        )
+    except Exception as e:
+        print(f"Error loading model: {e}")
+        return
+
+    data_dir = Path(__file__).parent.parent / "data"
+    if not data_dir.exists():
+        print(
+            f"Data directory {data_dir} not found. "
+            "Creating empty directory (add PDF files to it)."
+        )
+        data_dir.mkdir(parents=True, exist_ok=True)
+
+    pdf_files = list(data_dir.glob("*.pdf"))
+    if not pdf_files:
+        print(f"No PDF files found in {data_dir}.")
+        return
+
+    print(f"Found {len(pdf_files)} PDF files to process.")
+
+    def all_pdfs_generator() -> Generator[Dict[str, Any], None, None]:
+        """Generate processed data from all PDF files."""
+        for pdf_path in pdf_files:
+            print(f"Processing document: {pdf_path.name}")
+            # Extract chapter name from filename
+            chapter_name = pdf_path.stem
+            yield from prepare_text_for_rag(pdf_path, model, chapter_name=chapter_name)
+
+    output_sql = Path(__file__).parent / "init.sql"
+    generate_sql(all_pdfs_generator(), output_file=str(output_sql))
+
+
+if __name__ == "__main__":
+    main()

parser/nlp/__init__.py

@@ -0,0 +1,6 @@
+"""NLP processing module for text cleaning and keyword extraction."""
+
+from .text_processor import filter_text, split_into_blocks
+from .keyword_extractor import extract_keywords
+
+__all__ = ["filter_text", "split_into_blocks", "extract_keywords"]