Customizable questionsets and web app

.env.sample

@@ -0,0 +1 @@
+OPENAI_API_KEYS = ['sk-xxx',]

.gitignore

@@ -26,6 +26,11 @@ share/python-wheels/
 *.egg
 MANIFEST
 
+data/
+
+
+.env
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.

code/app.py

@@ -3,31 +3,43 @@
 from document import Report
 from reader import Reader
 from user_qa import UserQA
-#import cfg
 import webbrowser
 import asyncio
 from langchain.callbacks import get_openai_callback
+from utils import load_config, structure_data, generate_csv, load_all_data, combine_data
+import yaml
+
+from dotenv import load_dotenv
+
+# Load environment variables from a .env file
+load_dotenv()
+
 
-#TOP_K = cfg.retriever_top_k
 TOP_K = 20
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--pdf_path", type=str, default=None)
     parser.add_argument("--pdf_url", type=str, default=None)
-    parser.add_argument("--basic_info_dir", type=str, default='data/basic_info')
     parser.add_argument("--llm_name", type=str, default='gpt-3.5-turbo')
-    parser.add_argument("--answers_dir", type=str, default='data/answers')
-    parser.add_argument("--assessment_dir", type=str, default='data/assessment')
-    parser.add_argument("--vector_db_dir", type=str, default='data/vector_db')
-    parser.add_argument("--retrieved_chunks_dir", type=str, default='data/retrieved_chunks')
-    parser.add_argument("--user_qa_dir", type=str, default='data/user_qa')
     parser.add_argument("--user_question", type=str, default='')
     parser.add_argument("--answer_length", type=int, default=50)
     parser.add_argument("--detail", action='store_true', default=False)
     parser.add_argument("--top_k", type=int, default=20)
+    parser.add_argument("--question_set", type=str, default='default', help="Specify the question set to use")
     args = parser.parse_args()
 
+    with open(f'question_sets/{args.question_set}.yaml', 'r') as file:
+        question_set = yaml.safe_load(file)
+
+    base_dir = f"data/{args.question_set}"
+    basic_info_dir = os.path.join(base_dir, 'basic_info')
+    answers_dir = os.path.join(base_dir, 'answers')
+    assessment_dir = os.path.join(base_dir, 'assessment')
+    vector_db_dir = os.path.join(base_dir, 'vector_db')
+    retrieved_chunks_dir = os.path.join(base_dir, 'retrieved_chunks')
+    user_qa_dir = os.path.join(base_dir, 'user_qa')
+
     if args.pdf_path:
         report_name = os.path.basename(args.pdf_path)
     else:
@@ -36,18 +48,18 @@ def main():
     assert report_name.endswith('.pdf')
     report_name = report_name.replace('.pdf', '')
 
-    if not os.path.exists(args.basic_info_dir):
-        os.makedirs(args.basic_info_dir)
-    if not os.path.exists(args.answers_dir):
-        os.makedirs(args.answers_dir)
-    if not os.path.exists(args.assessment_dir):
-        os.makedirs(args.assessment_dir)
-    if not os.path.exists(args.vector_db_dir):
-        os.makedirs(args.vector_db_dir)
-    if not os.path.exists(args.retrieved_chunks_dir):
-        os.makedirs(args.retrieved_chunks_dir)
-    if not os.path.exists(args.user_qa_dir):
-        os.makedirs(args.user_qa_dir)
+    if not os.path.exists(basic_info_dir):
+        os.makedirs(basic_info_dir)
+    if not os.path.exists(answers_dir):
+        os.makedirs(answers_dir)
+    if not os.path.exists(assessment_dir):
+        os.makedirs(assessment_dir)
+    if not os.path.exists(vector_db_dir):
+        os.makedirs(vector_db_dir)
+    if not os.path.exists(retrieved_chunks_dir):
+        os.makedirs(retrieved_chunks_dir)
+    if not os.path.exists(user_qa_dir):
+        os.makedirs(user_qa_dir)
     destination_folder = "data/pdf/"
     if not os.path.exists(destination_folder):
         os.makedirs(destination_folder)
@@ -56,14 +68,14 @@ def main():
         path=args.pdf_path,
         url=args.pdf_url,
         store_path=os.path.join(destination_folder, report_name + '.pdf'),
-        db_path=os.path.join(args.vector_db_dir, report_name),
-        retrieved_chunks_path=os.path.join(args.retrieved_chunks_dir, report_name)
+        db_path=os.path.join(vector_db_dir, report_name),
+        retrieved_chunks_path=os.path.join(retrieved_chunks_dir, report_name),
+        question_set= question_set
     )
 
-    if args.user_question == '':
+    if True:
         try:
-            reader = Reader(llm_name=args.llm_name, answer_length=str(args.answer_length),)
-                            # qa_prompt="tcfd_summary_source", answer_key_name='SUMMARY', q_name='Q', a_name='Summary')
+            reader = Reader(llm_name=args.llm_name, answer_length=str(args.answer_length), question_set=question_set)
             result_qa = asyncio.run(reader.qa_with_chat(report_list=[report]))
             result_analysis = asyncio.run(reader.analyze_with_chat(report_list=[report]))
         except Exception as e:
@@ -76,11 +88,10 @@ def main():
                     path=os.path.join(destination_folder, args.pdf_path.split('/')[-1]),
                     store_path=None,
                     top_k=TOP_K - 5,
-                    db_path=os.path.join(args.vector_db_dir, report_name),
-                    retrieved_chunks_path=os.path.join(args.retrieved_chunks_dir, report_name),
+                    db_path=os.path.join(vector_db_dir, report_name),
+                    retrieved_chunks_path=os.path.join(retrieved_chunks_dir, report_name)
                 )
-                reader = Reader(llm_name=args.llm_name, answer_length=str(args.answer_length),)
-                                #qa_prompt="tcfd_summary_source", answer_key_name='SUMMARY', q_name='Q', a_name='Summary')
+                reader = Reader(llm_name=args.llm_name, answer_length=str(args.answer_length))
                 result_qa = asyncio.run(reader.qa_with_chat(report_list=[report]))
                 result_analysis = asyncio.run(reader.analyze_with_chat(report_list=[report]))
 
@@ -90,28 +101,39 @@ def main():
             f.write(result_qa[0])
         with open(html_path_analysis, 'w') as f:
             f.write(result_analysis[0])
-        # webbrowser.open(html_path)
-        with open(os.path.join(args.basic_info_dir, report_name + '_' + args.llm_name + '.json'), 'w') as f:
+        with open(os.path.join(basic_info_dir, report_name + '_' + args.llm_name + '.json'), 'w') as f:
             json.dump(reader.basic_info_answers[0], f)
-        with open(os.path.join(args.answers_dir, report_name + '_' + args.llm_name + '.json'), 'w') as f:
+        with open(os.path.join(answers_dir, report_name + '_' + args.llm_name + '.json'), 'w') as f:
             json.dump(reader.answers[0], f)
-        with open(os.path.join(args.assessment_dir, report_name + '_' + args.llm_name + '.json'), 'w') as f:
+        with open(os.path.join(assessment_dir, report_name + '_' + args.llm_name + '.json'), 'w') as f:
             json.dump(reader.assessment_results[0], f)
+        # Load all data
+        all_answers, all_assessments, retrieved_chunks = load_all_data(answers_dir, assessment_dir, retrieved_chunks_dir)
+
+        # Combine all data
+        combined_answers, combined_assessments = combine_data(all_answers, all_assessments)
+
+        # Structure data for CSV
+        structured_data = structure_data(report_name, combined_answers, combined_assessments, retrieved_chunks)
+
+        # Generate or update CSV file
+        csv_output_path = os.path.join(base_dir, f"{args.question_set}_answers_assessments.csv")
+        generate_csv(structured_data, csv_output_path)
     else:
         qa = UserQA(llm_name=args.llm_name)
         answer, _ = qa.user_qa(
             args.user_question,
             report,
-            basic_info_path=os.path.join(args.basic_info_dir, report_name + '_' + args.llm_name + '.json'),
+            basic_info_path=os.path.join(basic_info_dir, report_name + '_' + args.llm_name + '.json'),
             answer_length=args.answer_length,
         )
         print(answer)
-        with open(os.path.join(args.user_qa_dir, report_name + '_' + args.llm_name + '.jsonl'), 'a') as f:
+        with open(os.path.join(user_qa_dir, report_name + '_' + args.llm_name + '.jsonl'), 'a') as f:
             qa_json = json.dumps(answer)
             f.write(qa_json + '\n')
 
 
 if __name__ == '__main__':
     with get_openai_callback() as cb:
         main()
-        print(cb)
+        print(cb)

code/cfg.py

@@ -1,3 +1,10 @@
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+api_key = os.getenv('API_KEY')
+
+
 # key: topic, value: list of search key
 # temperature for generation
 temperature = 0.

code/config.py

@@ -0,0 +1,21 @@
+import yaml
+
+class Config:
+    _instance = None
+
+    def __new__(cls, file_path=None):
+        if cls._instance is None:
+            cls._instance = super(Config, cls).__new__(cls)
+            cls._instance.config = {}  # Initialize config attribute
+            if file_path:
+                cls._instance.load_config(file_path)
+        return cls._instance
+
+    def load_config(self, file_path):
+        import pdb; pdb.set_trace()
+
+        with open(file_path, 'r') as file:
+            self.config = yaml.safe_load(file)
+
+    def get_config(self):
+        return self.config

code/document.py

@@ -14,46 +14,34 @@
 from langchain.embeddings.openai import OpenAIEmbeddings
 import time
 import requests
-import configparser
 import os
 
-config = configparser.ConfigParser()
-config.read('apikey.ini')
-chat_api_list = config.get('OpenAI', 'OPENAI_API_KEYS')[1:-1].replace('\'', '').split(',')
-os.environ["OPENAI_API_KEY"] = chat_api_list[0]
+from dotenv import load_dotenv
+from config import Config
+
+# Load environment variables from a .env file
+load_dotenv()
+
+
 
 TOP_K = 20
 CHUNK_SIZE = 500
 CHUNK_OVERLAP = 20
 COMPRESSION = False
-QUERIES = {
-    'general': ["What is the company of the report?", "What sector does the company belong to?", "Where is the company located?",
-                #"What climate-related issues are discussed in this report?"
-          ],
-    'tcfd_1': "How does the company's board oversee climate-related risks and opportunities?",
-    'tcfd_2': "What is the role of management in assessing and managing climate-related risks and opportunities?",
-    'tcfd_3': "What are the most relevant climate-related risks and opportunities that the organisation has identified over the short, medium, and long term? Are risks clearly associated with a horizon?",
-    'tcfd_4': "How do climate-related risks and opportunities impact the organisation's businesses strategy, economic and financial performance, and financial planning?",
-    'tcfd_5': "How resilient is the organisation's strategy when considering different climate-related scenarios, including a 2°C target or lower scenario? How resilient is the organisation's strategy when considering climate physical risks?",
-    'tcfd_6': "What processes does the organisation use to identify and assess climate-related risks?",
-    'tcfd_7': "How does the organisation manage climate-related risks?",
-    'tcfd_8': "How are the processes for identifying, assessing, and managing climate-related risks integrated into the organisation's overall risk management?",
-    'tcfd_9': "What metrics does the organisation use to assess climate-related risks and opportunities? How do the metrics help ensure that the performance is in line with its strategy and risk management process?",
-    'tcfd_10': "Does the organisation disclose its Scope 1, Scope 2, and, if appropriate, Scope 3 greenhouse gas (GHG) emissions? What are the related risks and do they differ depending on the scope?",
-    'tcfd_11': "What targets does the organisation use to understand/quantify/benchmark climate-related risks and opportunities? How is the organization performing against these targets?",
-}
 
 
 class Report:
-    def __init__(self, path=None, url=None, title='', abs='', authers=[], store_path=None, top_k=TOP_K, db_path=None, retrieved_chunks_path=None):
+    def __init__(self, path=None, url=None, title='', abs='', authers=[], store_path=None, top_k=TOP_K, db_path=None, retrieved_chunks_path=None, question_set={}):
         # Init the class on pdf with given path
+        config = Config().get_config()
+
         self.chunks = []
         self.page_idx = []
         self.path = path  # pdf path
         self.url = url # pdf url
         assert ((path is None and url is not None) or (path is not None and url is None)) # only need to pass in an url or a path
         self.store_path = store_path
-        self.queries = QUERIES
+        self.queries = question_set['queries']
         self.top_k = top_k  # retriever top-k
         self.compression = COMPRESSION
         self.section_names = []  # title
@@ -176,8 +164,8 @@ def get_image_path(self, image_path=''):
     # recongnize the title by fontsize
     def get_chapter_names(self, ):
         # # open pdf
-        doc = fitz.open(self.path)
-        text_list = [page.get_text() for page in doc]
+        with fitz.open(self.path) as doc:
+            text_list = [page.get_text() for page in doc]
         all_text = ''
         for text in text_list:
             all_text += text
@@ -197,44 +185,44 @@ def get_chapter_names(self, ):
         return chapter_names
 
     def get_title(self):
-        doc = self.pdf
-        max_font_size = 0  # init fontsize 0
-        max_string = ""  # init max string
-        max_font_sizes = [0]
-        for page in doc:  # go through all pages
-            text = page.get_text("dict")  # acquire the info in the page
-            blocks = text["blocks"]  # acquire text block
-            for block in blocks:  # go through all blocks
-                if block["type"] == 0 and len(block['lines']):  # if str
-                    if len(block["lines"][0]["spans"]):
-                        font_size = block["lines"][0]["spans"][0]["size"]  # acquire fontsize
-                        max_font_sizes.append(font_size)
-                        if font_size > max_font_size:
-                            max_font_size = font_size  # update max fontsize
-                            max_string = block["lines"][0]["spans"][0]["text"]  # update the title str
-        max_font_sizes.sort()
-        print("max_font_sizes", max_font_sizes[-10:])
-        cur_title = ''
-        for page in doc:  # go through all pages
-            text = page.get_text("dict")
-            blocks = text["blocks"]
-            for block in blocks:
-                if block["type"] == 0 and len(block['lines']):
-                    if len(block["lines"][0]["spans"]):
-                        cur_string = block["lines"][0]["spans"][0]["text"]
-                        font_flags = block["lines"][0]["spans"][0]["flags"]
-                        font_size = block["lines"][0]["spans"][0]["size"]
-                        # print(font_size)
-                        if abs(font_size - max_font_sizes[-1]) < 0.3 or abs(font_size - max_font_sizes[-2]) < 0.3:
-                            # print("The string is bold.", max_string, "font_size:", font_size, "font_flags:", font_flags)
-                            if len(cur_string) > 4:
+        with fitz.open(self.path) as doc:
+            max_font_size = 0  # init fontsize 0
+            max_string = ""  # init max string
+            max_font_sizes = [0]
+            for page in doc:  # go through all pages
+                text = page.get_text("dict")  # acquire the info in the page
+                blocks = text["blocks"]  # acquire text block
+                for block in blocks:  # go through all blocks
+                    if block["type"] == 0 and len(block['lines']):  # if str
+                        if len(block["lines"][0]["spans"]):
+                            font_size = block["lines"][0]["spans"][0]["size"]  # acquire fontsize
+                            max_font_sizes.append(font_size)
+                            if font_size > max_font_size:
+                                max_font_size = font_size  # update max fontsize
+                                max_string = block["lines"][0]["spans"][0]["text"]  # update the title str
+            max_font_sizes.sort()
+            print("max_font_sizes", max_font_sizes[-10:])
+            cur_title = ''
+            for page in doc:  # go through all pages
+                text = page.get_text("dict")
+                blocks = text["blocks"]
+                for block in blocks:
+                    if block["type"] == 0 and len(block['lines']):
+                        if len(block["lines"][0]["spans"]):
+                            cur_string = block["lines"][0]["spans"][0]["text"]
+                            font_flags = block["lines"][0]["spans"][0]["flags"]
+                            font_size = block["lines"][0]["spans"][0]["size"]
+                            # print(font_size)
+                            if abs(font_size - max_font_sizes[-1]) < 0.3 or abs(font_size - max_font_sizes[-2]) < 0.3:
                                 # print("The string is bold.", max_string, "font_size:", font_size, "font_flags:", font_flags)
-                                if cur_title == '':
-                                    cur_title += cur_string
-                                else:
-                                    cur_title += ' ' + cur_string
-                                    # break
-        title = cur_title.replace('\n', ' ')
+                                if len(cur_string) > 4:
+                                    # print("The string is bold.", max_string, "font_size:", font_size, "font_flags:", font_flags)
+                                    if cur_title == '':
+                                        cur_title += cur_string
+                                    else:
+                                        cur_title += ' ' + cur_string
+                                        # break
+            title = cur_title.replace('\n', ' ')
         return title
 
     # def _get_all_page_index(self):
@@ -286,7 +274,7 @@ def _get_retriever(self, db_path):
             self.page_idx.extend([i + 1] * len(page_chunks))
             self.chunks.extend(page_chunks)
         if os.path.exists(db_path):
-            doc_search = FAISS.load_local(db_path, embeddings=embeddings)
+            doc_search = FAISS.load_local(db_path, embeddings=embeddings, allow_dangerous_deserialization=True)
         else:
             doc_search = FAISS.from_texts(self.chunks, embeddings,
                                           metadatas=[{"source": str(i), "page": str(page_idx)} for i, page_idx in
@@ -315,4 +303,3 @@ def search_page(content, search_list):
         return True
     else:
         return False
-