JavaDocRAG/config.py at main · sririthishpalani-max/JavaDocRAG · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
"""Central configuration – all tunable knobs live here."""
import os
from pathlib import Path

from dotenv import load_dotenv

load_dotenv()

# ── Paths ─────────────────────────────────────────────────────────────────────
DATA_DIR   = Path(os.getenv("DATA_DIR", "data"))
DB_PATH    = DATA_DIR / "db" / "javadoc.db"
FAISS_DIR  = DATA_DIR / "index"
BM25_PATH  = DATA_DIR / "bm25" / "bm25.pkl"
CACHE_DIR  = DATA_DIR / "html_cache"   # raw HTML cache (avoids re-fetching)

# ── Crawler ───────────────────────────────────────────────────────────────────
BASE_URL = "https://docs.oracle.com/en/java/javase/25/docs/api/"

# Seed pages that list every type / package / module
SEED_URLS: list[str] = [
    BASE_URL + "index.html",
    BASE_URL + "allpackages-index.html",
    BASE_URL + "allclasses-index.html",
    BASE_URL + "help-doc.html",
]

# Search-index JSON files exposed by the Javadoc toolchain –
# these give us structured type/member lists without crawling every page first.
SEARCH_INDEX_URLS: list[str] = [
    BASE_URL + "type-search-index.js",
    BASE_URL + "member-search-index.js",
    BASE_URL + "package-search-index.js",
    BASE_URL + "module-search-index.js",
]

MAX_CONCURRENT_REQUESTS = int(os.getenv("MAX_CONCURRENT", "5"))
REQUEST_DELAY_SECONDS   = float(os.getenv("REQUEST_DELAY", "1.0"))
MAX_PAGES               = int(os.getenv("MAX_PAGES", "0"))   # 0 = no cap
HTTP_TIMEOUT            = 30.0
USER_AGENT              = "JavaDocRAG-Ingestor/1.0 (educational; contact: see repo)"

# ── Chunking ──────────────────────────────────────────────────────────────────
MAX_CHUNK_TOKENS    = int(os.getenv("MAX_CHUNK_TOKENS", "800"))
CHUNK_OVERLAP_TOKENS = int(os.getenv("CHUNK_OVERLAP_TOKENS", "100"))

# ── Embedding ─────────────────────────────────────────────────────────────────
EMBEDDING_MODEL      = os.getenv("EMBEDDING_MODEL", "BAAI/bge-small-en-v1.5")
EMBEDDING_DIM        = int(os.getenv("EMBEDDING_DIM", "384"))  # bge-small=384, bge-base=768
EMBEDDING_BATCH_SIZE = int(os.getenv("EMBEDDING_BATCH_SIZE", "32"))
# Instruction prefix required by BGE models for queries
BGE_QUERY_INSTRUCTION = "Represent this sentence for searching relevant passages: "

# ── Reranker ──────────────────────────────────────────────────────────────────
RERANKER_MODEL = os.getenv(
    "RERANKER_MODEL", "cross-encoder/ms-marco-MiniLM-L-6-v2"
)

# ── Retrieval ─────────────────────────────────────────────────────────────────
BM25_TOP_K   = int(os.getenv("BM25_TOP_K", "50"))
DENSE_TOP_K  = int(os.getenv("DENSE_TOP_K", "50"))
RERANK_TOP_K = int(os.getenv("RERANK_TOP_K", "8"))
BM25_WEIGHT  = float(os.getenv("BM25_WEIGHT", "0.4"))
DENSE_WEIGHT = float(os.getenv("DENSE_WEIGHT", "0.6"))
RRF_K        = 60   # Reciprocal Rank Fusion constant

# ── Generation ────────────────────────────────────────────────────────────────
ANTHROPIC_API_KEY  = os.getenv("ANTHROPIC_API_KEY", "")
CLAUDE_MODEL       = os.getenv("CLAUDE_MODEL", "claude-sonnet-4-6")
MAX_CONTEXT_CHUNKS = int(os.getenv("MAX_CONTEXT_CHUNKS", "8"))
MAX_ANSWER_TOKENS  = 1024

# ── Server ────────────────────────────────────────────────────────────────────
HOST = os.getenv("HOST", "0.0.0.0")
PORT = int(os.getenv("PORT", "8000"))

# ── Image Support (Multimodal RAG) ─────────────────────────────────────────────
ENABLE_IMAGE_INGESTION = os.getenv("ENABLE_IMAGE_INGESTION", "true").lower() == "true"
ENABLE_IMAGE_EMBEDDING = os.getenv("ENABLE_IMAGE_EMBEDDING", "true").lower() == "true"
ENABLE_IMAGE_SEARCH = os.getenv("ENABLE_IMAGE_SEARCH", "true").lower() == "true"

IMAGE_EMBEDDING_MODEL = os.getenv("IMAGE_EMBEDDING_MODEL", "openai/clip-vit-base-patch32")
IMAGE_EMBEDDING_DIM = int(os.getenv("IMAGE_EMBEDDING_DIM", "512"))  # CLIP typically 512 or 768

# Storage
MAX_IMAGE_SIZE_MB = int(os.getenv("MAX_IMAGE_SIZE_MB", "5"))
EMBED_IMAGES_IN_DB = os.getenv("EMBED_IMAGES_IN_DB", "true").lower() == "true"

# Retrieval
IMAGE_TOP_K = int(os.getenv("IMAGE_TOP_K", "5"))
IMAGE_WEIGHT_IN_FUSION = float(os.getenv("IMAGE_WEIGHT_IN_FUSION", "0.2"))

# Server
IMAGE_UPLOAD_DIR = DATA_DIR / "uploads"
MAX_UPLOAD_SIZE_MB = int(os.getenv("MAX_UPLOAD_SIZE_MB", "10"))