Minor changes after some labelling process

Author: Orieus

.DS_Store

@@ -99,3 +99,6 @@ ENV/
 
 # mypy
 .mypy_cache/
+
+# Added by JCS
+*.DS_store

.gitignore

@@ -85,7 +85,7 @@ def compute_tpfn(p, NFP, NFN):
     return th, pos
 
 
-def plotROCs(p, y, w, rs0al1, relabels, category=None):
+def plotROCs(p, y, w, rs0al1, relabels, category=None, fpath=None):
 
     """ Plot three possible decision thresholds for a classifier with
         predictions p for samples with labels y, depending on the use of
@@ -146,9 +146,14 @@ def plotROCs(p, y, w, rs0al1, relabels, category=None):
     plt.plot(FPR[pos], TPR[pos], 'r.', markersize=10)
     plt.plot(FPRrs[pos_rs], TPRrs[pos_rs], 'g.', markersize=10)
     plt.plot(FPRw[pos_w], TPRw[pos_w], 'b.', markersize=10)
+    plt.axis('square')
+    plt.xlim((0, 1))
+    plt.ylim((0, 1))
     plt.show(block=False)
+    if fpath is not None:
+        plt.savefig(fpath)
 
-    # ROC based on sklearn (the results are assentially the same)
+    # ROC based on sklearn (the results are essentially the same)
     # from sklearn.metrics import roc_curve
     # FPR2, TPR2, tt = roc_curve(y_sorted, p_sorted)
     # FPRrs2, TPRrs2, tt = roc_curve(y_sorted2, p_sorted2)

labelfactory/.DS_Store

@@ -31,6 +31,8 @@
 else:
     import Tkinter as tk
 
+import ipdb
+
 # The following is to capture a user tag automatically.
 try:
     import pwd
@@ -75,14 +77,6 @@ def run_labeler(project_path, url, transfer_mode, user, export_labels,
                     labeling windows
     """
 
-    # To complete the migration to python 3, I should replace all "raw_input"
-    # by "input". Transitorily, to preserve compatibility with python 2, I
-    # simply rename inut to raw_input
-    if sys.version_info.major == 3:
-        raw_input2 = input
-    else:
-        raw_input2 = raw_input
-
     #######
     # Start
     #######
@@ -93,22 +87,17 @@ def run_labeler(project_path, url, transfer_mode, user, export_labels,
             tm_options))
 
     # # Check if project folder exists. Otherwise create a default one.
-    # if len(sys.argv) > 1:
-    #     project_path = sys.argv[1]
-    # else:
-    #     project_path = raw_input2("Select the (absolute or relative) path " +
-    #                               "to the labeling project folder: ")
     if project_path is None:
-        project_path = raw_input2("Select the (absolute or relative) path " +
-                                  "to the labeling project folder: ")
+        project_path = input("Select the (absolute or relative) path to " +
+                             "the labeling project folder: ")
 
     if not project_path.endswith('/'):
         project_path = project_path + '/'
 
     # Check if project folder exists. This is necessary to follow
     if not os.path.isdir(project_path):
-        createfolder = raw_input2("Folder " + project_path +
-                                  "does not exist. Create? (y/n)")
+        createfolder = input(f"Folder {project_path} does not exist. " + 
+                             "Create? (y/n)")
 
         if createfolder == "y":
             os.makedirs(project_path)    # Create folder
@@ -154,8 +143,7 @@ def run_labeler(project_path, url, transfer_mode, user, export_labels,
                         'DataPaths', 'db_label_info_tablename'),
                    'history_tablename': cf.get(
                         'DataPaths', 'db_history_tablename'),
-                   'ref_name': cf.get(
-                        'DataPaths', 'db_ref_name'),
+                   'ref_name': cf.get('DataPaths', 'db_ref_name'),
                    'mode': cf.get('DataPaths', 'db_mode')}
     elif source_type == 'mongodb' or dest_type == 'mongodb':
         db_info = {'name': cf.get('DataPaths', 'db_name'),
@@ -287,14 +275,13 @@ def run_labeler(project_path, url, transfer_mode, user, export_labels,
                     user0 = getpass.getuser()  # For Windows
 
                 # Ask for a labeler name.
-                userId = raw_input2(
-                    "Please write you user name [{0}]: ".format(user0))
+                userId = input(f"Please write you user name [{user0}]: ")
                 if userId == "":
                     userId = user0
-                    print("Your User Name is {0}".format(userId))
+                    print(f"Your User Name is {userId}")
             else:
                 userId = user
-                print("Your User Name is {0}".format(userId))
+                print(f"Your User Name is {userId}")
         else:
             userId = None
             if user is not None:
@@ -309,7 +296,7 @@ def run_labeler(project_path, url, transfer_mode, user, export_labels,
     # Verify that parentclass names are in the category set
     for c in parentcat:
         if c not in categories:
-            log.error("Unknown category {} ".format(c) + "in the parentcat " +
+            log.error(f"Unknown category {c} in the parentcat " +
                       "dictionary. Revise the configuration file")
             sys.exit()
         elif parentcat[c] not in categories:

labelfactory/dataanalyzer/.DS_Store

@@ -565,7 +565,7 @@ def importPredicts(self, category=None):
             # Load predictions from file, if it exists
             if os.path.isfile(preds_file):
                 print("---- Importing predictions from category " + category)
-                with open(preds_file, 'r') as f:
+                with open(preds_file, 'rb') as f:
                     preds = pickle.load(f)
 
                 # Move the preditions file to the "used" folder.
@@ -653,6 +653,7 @@ def computeWID(self, url, mode='yes'):
             # This replacement does not affect the url if it is a domain site.
             # But it may transform the url of specific web pages.
             wid = wid.replace("/", "__")
+
         elif mode == 'www':
 
             if url[0:4] == 'www.':

labelfactory/dataanalyzer/ROCanalyzer.py

@@ -22,6 +22,18 @@
 # Services from the project
 # sys.path.append(os.getcwd())
 
+# #############################################################################
+# #############################################################################
+# #############################################################################
+#
+# WARNING:
+#     THIS CLASS IS DEPRECATED. IT HAS BEEN REPLACED BY baseDM.py AND
+#     THEIR INHERITED CLASSES. HOWEVER, SOME OLD SCRIPT ARE STILL USING THIS
+#     CLASS. SO IT IS PRESERVED.
+# 
+# #############################################################################
+# #############################################################################
+# #############################################################################
 
 class DataManager(object):
 
@@ -315,6 +327,8 @@ def loadData(self):
                 # Load prediction dataframes stored in pickle files
                 df_preds = pd.read_pickle(self.datapreds_file)
 
+        ipdb.set_trace()
+
         return df_labels, df_preds, labelhistory
 
     def get_df(self, data, labelhistory):
@@ -790,6 +804,7 @@ def computeWID(self, url, mode='yes'):
             # This replacement does not affect the url if it is a domain site.
             # But it may transform the url of specific web pages.
             wid = wid.replace("/", "__")
+
         elif mode == 'www':
 
             if url[0:4] == 'www.':

labelfactory/labelfactory.py

@@ -142,6 +142,22 @@ def loadData(self):
             df_preds = pd.DataFrame(index=[], columns=cols)
             print(df_preds)
 
+        # df_preds = df_preds[pd.notnull(df_preds['b2c_on'])]
+
+        # df_labels.rename(
+        #     index={x: self.computeWID(x) for x in df_labels.index},
+        #     inplace=True)
+        # df_preds.rename(
+        #     index={x: self.computeWID(x) for x in df_preds.index},
+        #     inplace=True)
+        # labelhistory = {self.computeWID(x): labelhistory[x]
+        #                 for x in labelhistory}
+
+        # df_labels.replace({'O++t++h++e++r': 'Other'}, inplace=True)
+        # df_labels.replace({'b++2++c++_++o++n': 'b2c_on'}, inplace=True)
+        # df_labels.replace({'b++2++c++_++r++e++a++d++y': 'b2c_ready'},
+        #                   inplace=True)
+
         return df_labels, df_preds, labelhistory
 
     def saveData(self, df_labels, df_preds, labelhistory, save_preds=True):