Merge pull request #23 from TheBluCoder/dev-branch

Dev branch

Author: TheBluCoder

.gitignore

@@ -59,3 +59,4 @@ unit_test.py
 testing_workflow.py
 *.yaml
 
+scripts/

backend/mainService/Dockerfile

@@ -7,6 +7,7 @@ WORKDIR /app
 # Removes the package lists downloaded during the update to reduce the image size.
 RUN apt-get update && apt-get install -y \
     build-essential \
+    cron \
     && rm -rf /var/lib/apt/lists/*
 
 # Set the PATH environment variable to include /app
@@ -33,6 +34,5 @@ RUN playwright install && playwright install-deps
 # Expose the port the app runs on
 EXPOSE 8000
 
-
-# Command to run the application
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] 
+# Start both cron and the FastAPI application
+CMD ["sh", "-c", "cron && uvicorn app:app --host 0.0.0.0 --port 8000"] 

backend/mainService/requirements.txt

@@ -25,4 +25,6 @@ google-genai
 redis>=4.2.0
 uvicorn
 httpx>=0.28.1
+pypdf
+pypdf2
 

backend/mainService/scripts/delete_stale_data.py

@@ -32,7 +32,7 @@ class ScraperConfig:
     """
     This is the timeout duration for the requests made to the web scraper
     """
-    TIMEOUT_DURATION: int = 8000
+    TIMEOUT_DURATION: int = 10000
 
     def __post_init__(self):
         if self.MAX_FILE_SIZE <= 0:

backend/mainService/src/config/config.py

@@ -186,14 +186,14 @@ async def set_current_index(
 
         :param index_name: Name of the index to set as current
         """
+        if not await self._pc.has_index(index_name):
+                return False
         if not self._current_index_name == index_name and self._current_index:
             await self._current_index.close()
         elif self._current_index_name == index_name:
             return True
 
         if not index_host:
-            if not await self._pc.has_index(index_name):
-                return False
             index_model = await self._pc.describe_index(index_name)
             self._current_index_host = index_model.host
         else:

backend/mainService/src/llm/Pinecone.py

@@ -11,7 +11,7 @@
 
 
 class Genai_cite:
-    model = "gemini-2.0-pro-exp-02-05"
+    model = "gemini-2.0-flash"
 
     def __init__(self, api_key: str = os.getenv("GOOGLE_API_KEY"),
                  llm_model: str = f'models/{model}'):

backend/mainService/src/llm/chat_llm/Gemini_llm.py

@@ -19,7 +19,7 @@ def __init__(self, api_key: str = os.getenv("GROQ_API_KEY"),
         self.client = Groq(api_key=self.api_key)
         self.llm_model = llm_model
 
-    def getKeywordSearchTerm(self, document: str) -> Optional[str]:
+    def getKeywordSearchTerm(self, document: str, proposed_title: Optional[str] = None) -> str:
         """
         Generate a search term from the provided document using LLM.
 
@@ -46,12 +46,17 @@ def getKeywordSearchTerm(self, document: str) -> Optional[str]:
 
             # Make API call with error handling
 
+            if proposed_title:
+                document = f"Here is the proposed title: {proposed_title}\n\nHere is the content: {document}"
+            else:
+                document = f"Here is the content: {document}"
+
             completion = self.client.chat.completions.create(
                 model=self.llm_model,
                 messages=[
                     {
                         "role": "user",
-                        "content": f"summarize the provided into a google search term and return a json response as 'search_term : value', if no content provided, your response should be 'message:no content to summarize'. Here is the content: {document}"
+                        "content": f"summarize the provided into a google search term and return a json response as 'search_term : value', if no content provided, your response should be 'message:no content to summarize'.{document}"
                     },
                 ],
                 temperature=0.9,

backend/mainService/src/llm/chat_llm/Groq_llm.py

@@ -38,7 +38,7 @@ async def _get_download_link(self, url: str) -> Optional[str]:
         try:
             page = await self.context.new_page()
             if not url.endswith("pdf"):
-                await page.goto(url, wait_until='networkidle')
+                await page.goto(url, wait_until='networkidle', timeout=self.element_timeout)
                 await self._interact_with_dropdown(page)
                 download_link = await self._extract_download_link(page)
             else:

backend/mainService/src/scraper/site_specific/async_frontier_scraper.py

@@ -130,8 +130,7 @@ async def process_citation(self,
         """
         try:
             # Step 0: Generate index name
-            title = (self.summarize_llm.getKeywordSearchTerm(content)
-                     if title.lower() == "untitled" else title)
+            title = self.summarize_llm.getKeywordSearchTerm(content, proposed_title=title)
             index_name = self._generate_index_name(title)
             logger.info(f"index_name = {index_name}")
             if await self.PC.set_current_index(index_name):
@@ -230,10 +229,11 @@ async def _process_documents(
 
         try:
             cleaned_result = search_results["cleaned_result"]
-            download_results = await self.scraper.get_pdfs(
-                target_urls=cleaned_result.get("links"),
-                storage_path=search_results["search_key"]
-            )
+            async with asyncio.timeout(15):  # 15 second timeout
+                download_results = await self.scraper.get_pdfs(
+                    target_urls=cleaned_result.get("links"),
+                    storage_path=search_results["search_key"]
+                )
 
             return await self._prepare_document_batches(
                 download_results,