ETL Pipeline for Multi-Source Bibliographic Data

.gitignore

@@ -1,4 +1,5 @@
 __pycache__/
 bibliovenv/
 Bibenv/
-.idea/
+.idea/venv/
+venv/

functions/get_frequentwords.py

@@ -1,5 +1,5 @@
 from www.services import *
-
+import ast
 
 def get_frequent_words(df, ngram, num_of_words, word_type, file_upload_terms, file_upload_synonyms, field_separator_frequent=';'):
     """
@@ -116,7 +116,13 @@ def table_tag(df, tag, ngrams=1, remove_terms=None, synonyms=None):
 
     # Handle list columns (DE and ID)
     if tag in ['DE', 'ID']:
-        text_data = text_data.dropna().apply(lambda x: ', '.join(eval(x) if isinstance(x, str) else x))
+        text_data = text_data.dropna().apply(
+            lambda x: ', '.join(
+                x if isinstance(x, list)
+                else (ast.literal_eval(x) if isinstance(x, str) and x.strip().startswith("[")
+                      else [i.strip() for i in x.split(";")])
+            )
+        )
 
     # Process words
     if tag in ['DE', 'ID']:

functions/get_relevantauthors.py

@@ -1,5 +1,5 @@
 from www.services import *
-
+import ast
 
 def get_relevant_authors(df, num_of_authors, frequency="N. of Documents"):
     """
@@ -19,8 +19,12 @@ def get_relevant_authors(df, num_of_authors, frequency="N. of Documents"):
     data = data.dropna(subset=["AU"])
 
     # Ensure all values in the "AU" column are lists
-    data["AU"] = data["AU"].apply(lambda x: x if isinstance(x, list) else [])
-
+    #data["AU"] = data["AU"].apply(lambda x: x if isinstance(x, list) else [])
+    data["AU"] = data["AU"].apply(
+        lambda x: x if isinstance(x, list)
+        else (ast.literal_eval(x) if isinstance(x, str) and x.strip().startswith("[")
+              else ([i.strip() for i in x.split(";")] if isinstance(x, str) and x.strip() else []))
+    )
     # Flatten the list of authors and calculate occurrences
     all_authors = [author for sublist in data["AU"] for author in sublist]
     author_counts = pd.Series(all_authors).value_counts()
@@ -103,7 +107,10 @@ def get_relevant_authors(df, num_of_authors, frequency="N. of Documents"):
         )
 
     # Set x-axis ticks to 0, 5, 10, etc.
+
     max_x = author_counts[frequency].max()
+    if pd.isna(max_x):
+        max_x = 0
     tick_step = 5
     x_ticks = list(range(0, int(max_x) + tick_step, tick_step))
     if x_ticks[-1] < max_x:

functions/get_wordcloud.py

@@ -1,5 +1,5 @@
 from www.services import *
-
+import ast
 
 def is_legible_on_white(color):
     """Restituisce True se il colore è leggibile su sfondo bianco"""
@@ -122,7 +122,14 @@ def table_tag(df, tag, ngrams=1, remove_terms=None, synonyms=None):
 
     # Handle list columns (DE and ID)
     if tag in ['DE', 'ID']:
-        text_data = text_data.dropna().apply(lambda x: ', '.join(eval(x) if isinstance(x, str) else x))
+        #text_data = text_data.dropna().apply(lambda x: ', '.join(eval(x) if isinstance(x, str) else x))
+        text_data = text_data.dropna().apply(
+            lambda x: ', '.join(
+                x if isinstance(x, list)
+                else (ast.literal_eval(x) if isinstance(x, str) and x.strip().startswith("[")
+                      else [i.strip() for i in x.split(";")])
+            )
+        )
 
     # Process words
     if tag in ['DE', 'ID']: