tempCodeRunnerFile.py

from langchain_community.document_loaders import PyPDFLoader, DirectoryLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
# from langchain_huggingface.embeddings import HuggingFaceEmbeddings
from langchain_core.documents import Document
from dotenv import load_dotenv
from pinecone import Pinecone, ServerlessSpec
from langchain_pinecone import PineconeVectorStore
from l
import os
load_dotenv()
index_name="chatbot"
PINE_CONE_API_KEY=os.getenv("PINE_CONE_API_KEY")
os.environ["PINE_CONE_API_KEY"]=PINE_CONE_API_KEY
pc=Pinecone(api_key=PINE_CONE_API_KEY)
from typing import List

# Step 1 load doc
DATA_PATH="data/"
def load_pdf_files(data):
    loader=DirectoryLoader(data,
    glob="*.pdf",
    loader_cls=PyPDFLoader)

    document=loader.load()
    return document
documents=load_pdf_files(DATA_PATH)
print(f"Length of document : {len(documents)}")

# Step 2 splitting/chunking
def filter_doc(docs:List[Document])->List[Document]:
    minimal_doc:List[Document]=[]
    for doc in docs:
        pg_label=doc.metadata.get("page_label")
        minimal_doc.append(
            Document(
                page_content=doc.page_content,
                metadata={"page_num":pg_label}
            )
        )
    return minimal_doc
minimal_doc=filter_doc(documents)
def split_document(extracted_doc):
    splitter=RecursiveCharacterTextSplitter(chunk_size=500,chunk_overlap=50)
    texts=splitter.split_documents(extracted_doc)
    return texts
text_chunks=split_document(minimal_doc)
print("Length of text chunks ", len(text_chunks))
# print(text_chunks[0])
# text_only=[text_chunk.page_content for text_chunk in text_chunks]
# metadatas=[{'page':doc.metadata['page_label']} for doc in text_chunks]
def clean_text(text: str) -> str:
    return text.encode("utf-8", errors="ignore").decode("utf-8")

records=[]
for i,d in enumerate(text_chunks):
    records.append({
        "id":f"Chunk {i+1}",
        "chunk_text":clean_text(d.page_content),
        "metadata": d.metadata["page_num"] 
    })

# Step 3 embedding model


# def get_embedding_model():
#     embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
#     return embedding_model
# embedding_model=get_embedding_model()

index_name = "chatbot"

if not pc.has_index(index_name):
    pc.create_index_for_model(
        name=index_name,
        cloud="aws",
        region="us-east-1",
        embed={
            "model":"llama-text-embed-v2",
            "field_map":{"text": "chunk_text"}
        }
    )
dense_index = pc.Index(index_name)


BATCH_SIZE=96
for i in range(0,len(records),BATCH_SIZE):
    batch=records[i:i+BATCH_SIZE]
    dense_index.upsert_records("chatbot", batch)
    print(f"Upserted {i}:{i+BATCH_SIZE} from records")
# # Step 4 store in vector store
vector_store = PineconeVectorStore(index=dense_index)
# DB_FAISS_PATH="vectorstore\db_faiss"
# db=FAISS.from_documents(text_chunks,embedding_model)
# db.save_local(DB_FAISS_PATH)