From b3b288c5210e36438e3c02a398b90de345bed437 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 22 Jun 2026 15:12:15 -0600 Subject: [PATCH 01/81] Fixed bug where path to index.html is in wrong place --- app/routes/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routes/routes.py b/app/routes/routes.py index b48ffc4..5bd2b07 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -164,4 +164,4 @@ def reset_app(): @app.get("/") def home(): - return send_file("../../ui/dist/index.html") + return send_file("../ui/dist/index.html") From 9a86e69e14e13cc1a38992a5f3896a4ec194a827 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 26 Jun 2026 16:14:58 -0600 Subject: [PATCH 02/81] Created data_attributes script for data profile class --- app/ai_utils/data_attributes.py | 250 ++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 app/ai_utils/data_attributes.py diff --git a/app/ai_utils/data_attributes.py b/app/ai_utils/data_attributes.py new file mode 100644 index 0000000..f72c914 --- /dev/null +++ b/app/ai_utils/data_attributes.py @@ -0,0 +1,250 @@ +import numpy as np +import pandas as pd +import json + +from app.server_utils.service_helpers import is_categorical, get_error_dist +from app.db_utils.execute_sql import fetch_sql + + + +# TODO: change name to data profile instead of ai_utils / related +# TODO: Make this work with this with server_utils/data_attribute_summary_integration.py to create a data profile class that handles this shit +# This one would just be data profile class while ^^^^ uses it to get the data attribute summaries to show stuff + +# maybe move this to a utils script or something? +def load_table_to_df(table_name, engine): + rows = fetch_sql(f'SELECT * FROM "{table_name}"', False, engine) + cols = fetch_sql(f"SELECT column_name FROM information_schema.columns WHERE table_name = '{table_name}' ORDER BY ordinal_position", False, engine) + df = pd.DataFrame(rows, columns=[row[0] for row in cols] if cols else None) + + return df + + +class DataProfile: + # TODO: add a way to figure out where the changes were made (more efficient updating of data attributes + def __init__(self, table_name, engine=None, main_df=None, error_df=None): + self.table_name = table_name + self.engine = engine + self.main_df = main_df + self.error_df = error_df + self.name_to_func = { + 'mean': self.calculate_mean, + 'median': self.calculate_median, + 'min': self.calculate_min, + 'max': self.calculate_max #TODO: add more + } + self.attribute_type_assignment = { # TODO: fix this name it sucks lol + 'numeric': ['mean', 'median', 'min', 'max'], + 'categorical': ['n_categories' + 'mode'] + } + + + if self.main_df is None: + assert engine is not None, f"engine cannot be None if main_df is None" + self.main_df = load_table_to_df(table_name, engine) + + if self.error_df is None: + assert engine is not None, f"engine cannot be None if error_df is None" + self.error_df = load_table_to_df(f"errors_{table_name}", engine) + + def look_up_stat_from_profile(self, attribute_name, column_name): + + + def query_summary_stat_from_main_df(self, stat_query, column_name): + # TODO: fix this probably + query = f'SELECT {stat_query}({column_name}) FROM "{self.table_name}"' + stat = fetch_sql(query, False, self.engine) + return stat + + + def calculate_column_attribute(self, attribute_name, column_name): + look_up_value = self.look_up_stat(self, attribute_name, column_name) + # TODO: add asserts to make sure that attribute is compatible with the type of the column + if look_up_value is not None: + return look_up_value + + calculate_attribute_func = self.name_to_func[attribute_name] + return calculate_attribute_func(column_name) + + # TODO: save stat off to table if newly calculated + def calculate_mean(self, column_name): + try: + avg = self.query_summary_stat_from_main_df('AVG', column_name) + except Exception as e: + print(f"Error fetching the mean for table {self.table_name} at column {column_name}: {e}") + + print("Calculating mean manually using data...") + + avg = self.main_df[column_name].mean() + print(f"Updating mean value for column {column_name} in data profile") + + return avg + + + def calculate_median(self, column_name): + try: + median = self.query_summary_stat_from_main_df('MEDIAN', column_name) + except Exception as e: + print(f"Error fetching the median for table {self.table_name} at column {column_name}: {e}") + + print("Calculating median manually using data...") + + median = self.main_df[column_name].median() + + print(f"Updating median value for column {column_name} in data profile") + + return median + + + def calculate_max(self, column_name): + try: + maximum = self.query_summary_stat_from_main_df('MAX', column_name) + except Exception as e: + print(f"Error fetching the maximum for table {self.table_name} at column {column_name}: {e}") + + print("Calculating maximum manually using data...") + + maximum = self.main_df[column_name].max() + + print(f"Updating maximum value for column {column_name} in data profile") + return maximum + + def calculate_min(self, column_name): + try: + minimum = self.query_summary_stat_from_main_df('MIN', column_name) + except Exception as e: + print(f"Error fetching the minimum for table {self.table_name} at column {column_name}: {e}") + + print("Calculating minimum manually using data...") + + minimum = self.main_df[column_name].min() + + print(f"Updating minimum value for column {column_name} in data profile") + + return minimum + + def calculate_num_categories(self, column_name): + try: + query = f'SELECT COUNT(DISTINCT "{column_name}") FROM "{self.table_name}"' + n_categories = fetch_sql(query, False, self.engine) + except Exception as e: + print(f"Error fetching the n_categories for table {self.table_name} at column {column_name}: {e}") + + print("Calculating n_categories manually using data...") + + n_categories = self.main_df[column_name].nunique() + + print(f"Updating n_categories value for column {column_name} in data profile") + + return n_categories + + + def calculate_mode(self, column_name): + print("Calculating mode manually using data...") + mode = self.main_df[column_name].mode() + + return mode + + + + + + + + + + + + + + + + + + + +''' + + + +# Gets the data attributes that are given to LLM +def get_data_attributes(tablename): + from app import engine + + # TODO: already a similar loop in data_attribute_summary_integration.py + # line 85, maybe somehow combine these? Idk if it makes sense to put ai + # related things in there though? + # build_attribute_distributions() very similar; combine into same function in future + main_rows = fetch_sql(f'SELECT * FROM "{tablename}"', False, engine) + + main_cols = fetch_sql( + f"SELECT column_name FROM information_schema.columns WHERE table_name = '{tablename}' ORDER BY ordinal_position", + False, engine) + + main_df = pd.DataFrame(main_rows, columns=[row[0] for row in main_cols] if main_cols else None) + + data_attributes = {} + for col in main_df.columns: + data_attributes[col] = get_col_attributes(main_df, col) + + return data_attributes + +def save_data_attributes(tablename, json_path="data_attributes.json"): + data_attributes = get_data_attributes(tablename) + with open(f'app/ai_utils/{json_path}', 'w') as outfile: + json.dump(data_attributes, outfile) + +# TODO: maybe use scipy so it's more efficient? +def get_median_absolute_deviation(col_data, median): + median_residuals = col_data - median + abs_median_residuals = np.abs(median_residuals) + mad = int(np.median(abs_median_residuals)) + return mad + +# Returns a list: [lower_fence, upper_fence] +def get_tukeys_fences(col_data, iqr): + q1 = np.percentile(col_data, 25) + q3 = np.percentile(col_data, 75) + lower_fence = (q1 - 1.5 * (iqr)).item() + upper_fence = (q3 + 1.5 * (iqr)).item() + + return (lower_fence, upper_fence) + +def get_col_attributes(df, col): + col_attributes = {} + col_data = df[col] + + col_attributes['data_type'] = str(col_data.dtype) + col_attributes['num_na'] = int(col_data.isnull().sum()) + col_attributes['count'] = int(col_data.count()) + + if is_categorical(col_data): + col_data = df[col].fillna('N/A') + col_attributes['num_unique'] = int(col_data.nunique()) + # TODO: maybe do proportion instead? + col_attributes['category_count'] = col_data.value_counts().to_dict() + else: + col_data = pd.to_numeric(df[col], errors='coerce').dropna() + col_attributes['mean'] = col_data.mean().item() + col_attributes['median'] = col_data.median().item() + col_attributes['min'] = col_data.min().item() + col_attributes['max'] = col_data.max().item() + # ik a lot of these do the same things but idk which one to choose + # --- measures of dispersion + col_attributes['var'] = col_data.var().item() + col_attributes['iqr'] = (col_data.quantile(0.75) - col_data.quantile(0.25)).item() + col_attributes['std'] = col_data.std().item() + col_attributes['skew'] = col_data.skew().item() + col_attributes['median_absolute_deviation'] = get_median_absolute_deviation(col_data, col_attributes['median']) + # ------- + + # List where first val is lower fence, second is upper fence + # Can't be tuple bc jsons don't support tuples + col_attributes['tukeys_fence'] = get_tukeys_fences(col_data, col_attributes['iqr']) + return col_attributes + + + + +''' \ No newline at end of file From 869a8e56698246da437bb9e048b191c989c2e18b Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 29 Jun 2026 13:17:22 -0600 Subject: [PATCH 03/81] Created function that generates data profile dataframe --- app/server_utils/service_helpers.py | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index c4a9e52..d22e3b9 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -175,6 +175,41 @@ def run_detectors(data_frame): frames = [anomaly_df, incomplete_df, missing_value_df,datatype_mismatch_df] return perform_melt(frames) +# TODO: modify this so it only updates the changed columns +def create_data_profile_df(table_name, engine, error_df, columns_to_include=None): + + data_profile = DataProfile(table_name, engine=engine, error_df=error_df) + if columns_to_include is None: + columns_to_include = data_profile.main_df.columns + + col_attribute_list = [] + + for col in columns_to_include: + + row_dict = {'column_name': col} + for attribute in data_profile.default_attributes: + if attribute not in data_profile.attribute_type_assignment['categorical'] and attribute not in data_profile.attribute_type_assignment['numeric']: + # Attribute doesn't exist in either categorical and numeric + print(f"ERROR: INVALID ATTRIBUTE {attribute}") + print("Skipping this attribute") + elif (is_categorical(data_profile.main_df[col]) and attribute not in data_profile.attribute_type_assignment['categorical']) or \ + (not is_categorical(data_profile.main_df[col]) and attribute not in data_profile.attribute_type_assignment['numeric']): + row_dict[attribute] = None + + # default attribute does is not compatible with column + continue + + print("CALCULATING ATTRIBUTE: ", attribute) + print("COLUMN: ", col) + + row_dict[attribute] = data_profile.calculate_column_attribute(attribute, col, False) + col_attribute_list.append(row_dict) + + df = pd.DataFrame(col_attribute_list) + + return df + + def calculate_attribute_rankings(error_df): """ Calculate attribute rankings by total error count From 13258ecbe49cd5596170c500bb5d9a05f218976d Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 29 Jun 2026 13:19:12 -0600 Subject: [PATCH 04/81] Modified update_errors_table so it returns the error_df so it can be passed into update_data_profile_table --- app/routes/wrangler_routes_sql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index 062364b..f6adc4a 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -35,6 +35,7 @@ def update_errors_table(table_name: str) -> None: conn.execute(sa_text(f'DROP TABLE IF EXISTS "{errors_table_name}"')) detected_errors_df.to_sql(errors_table_name, engine, if_exists='fail', index=False) print(f"✓ Updated errors table: {errors_table_name}") + return detected_errors_df except Exception as e: print(f"ERROR: Could not update errors table for {table_name}: {e}") traceback.print_exc() From 3a35c5cc5eda7e2f42d042883ff55d1bd6b6940c Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 29 Jun 2026 13:19:25 -0600 Subject: [PATCH 05/81] Modified update_errors_table so it returns the error_df so it can be passed into update_data_profile_table --- app/routes/wrangler_routes_sql.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index f6adc4a..ac2d981 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -20,7 +20,9 @@ # Helper: Re-run error detection after modification # ───────────────────────────────────────────────────────────────────────────── -def update_errors_table(table_name: str) -> None: +# Returns error_df for update_data_profile_table to use (so it doesn't have to get it from the database) +def update_errors_table(table_name: str) -> pd.DataFrame: + # TODO: fix this so it doesn't update the whole table after small changes to the table """ After modifying a table in-place, re-run error detection and update the errors table. From ac33f76bbdde599c9b4ad5709fda6847a0774e13 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 29 Jun 2026 13:19:56 -0600 Subject: [PATCH 06/81] Created update_data_profile_table --- app/routes/wrangler_routes_sql.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index ac2d981..059c919 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -43,6 +43,23 @@ def update_errors_table(table_name: str) -> pd.DataFrame: traceback.print_exc() raise +def update_data_profile_table(table_name: str, error_df: pd.DataFrame) -> None: + try: + data_profile_df = create_data_profile_df(table_name, engine, error_df) + print("CALCULATED DATA PROFILE DF SUCCESSFULLY:") + print(data_profile_df) + data_profile_table_name = f"dp_{table_name}" + with engine.begin() as conn: + conn.execute(sa_text(f"DROP TABLE IF EXISTS {data_profile_table_name}")) + data_profile_df.to_sql(data_profile_table_name, engine, if_exists='fail', index=False) + + print(f"✓ Updated data profile table: {data_profile_table_name}") + except Exception as e: + print(f"ERROR: Could not update data profile table for {table_name}: {e}") + traceback.print_exc() + raise + + def update_preview_error_table(table_name: str, err_table_name: str) -> None: """ After modifying a table in-place, re-run error detection From dd861b40d6b514c9f03b03ce025c77fd76447e79 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 29 Jun 2026 13:20:25 -0600 Subject: [PATCH 07/81] Imported create_data_profile_df into wrangler_routes_sql.py --- app/routes/wrangler_routes_sql.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index 059c919..5dbd622 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -7,7 +7,8 @@ from app import engine import traceback import pandas as pd -from app.server_utils.service_helpers import run_detectors, create_previews_1d, create_previews_2d, execute_wrangle_preview, _safe_pg_name +from app.server_utils.service_helpers import run_detectors, create_previews_1d, create_previews_2d, \ + execute_wrangle_preview, _safe_pg_name, create_data_profile_df from sqlalchemy import text as sa_text @@ -60,6 +61,15 @@ def update_data_profile_table(table_name: str, error_df: pd.DataFrame) -> None: raise +# TODO: Finish this later +#def update_stat_to_data_profile_table(table_name: str, error_df: pd.DataFrame) -> None: + + + + + + +# TODO: does this even do anything? Can I remove it? def update_preview_error_table(table_name: str, err_table_name: str) -> None: """ After modifying a table in-place, re-run error detection From ebac2a9b1ad86ce9604879570778ed32650f109e Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 29 Jun 2026 13:21:28 -0600 Subject: [PATCH 08/81] Updated create_previews_1d and create_previews_2d so they both update the data profile table --- app/routes/wrangler_routes_sql.py | 4 ++-- app/server_utils/service_helpers.py | 20 +++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index 5dbd622..914d919 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -132,9 +132,9 @@ def create_previews(): return {"success": False, "error": "No rows selected"}, 400 if len(cols) == 1: - return create_previews_1d(table, row_ids, cols, _safe_pg_name, update_errors_table) + return create_previews_1d(table, row_ids, cols, _safe_pg_name, update_errors_table, update_data_profile_table) else: - return create_previews_2d(table, row_ids, cols, _safe_pg_name, update_errors_table) + return create_previews_2d(table, row_ids, cols, _safe_pg_name, update_errors_table, update_data_profile_table) except Exception as e: print("ERROR in create_previews") diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index d22e3b9..b995976 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -481,7 +481,7 @@ def create_minimal_preview_table(conn, source_table, preview_table_name, errors_ # Copy schema but leave empty. conn.execute(sa_text(f'CREATE TABLE "{errors_dest}" (LIKE "{errors_source}" INCLUDING ALL)"')) -def create_previews_1d(table, row_ids, cols, preview_name_fn, update_errors_fn): +def create_previews_1d(table, row_ids, cols, preview_name_fn, update_errors_fn, update_data_profile_table_fn): """ Create delete and impute preview tables for a 1D (single-column) selection. Returns a dict with preview table names and dims=1. @@ -502,8 +502,10 @@ def create_previews_1d(table, row_ids, cols, preview_name_fn, update_errors_fn): query.remove_rows_by_ids(table=preview_delete, ids=row_ids) query.impute_by_ids(table=preview_impute, col=cols[0], ids=row_ids) - update_errors_fn(preview_delete) - update_errors_fn(preview_impute) + errors_df_delete = update_errors_fn(preview_delete) + errors_df_impute = update_errors_fn(preview_impute) + update_data_profile_table_fn(preview_delete, errors_df_delete) + update_data_profile_table_fn(preview_impute, errors_df_impute) return { "success": True, @@ -520,7 +522,7 @@ def extract_preview_action(name: str) -> str: return name[idx + len(marker):] return "" -def create_previews_2d(table, row_ids, cols, preview_name_fn, update_errors_fn): +def create_previews_2d(table, row_ids, cols, preview_name_fn, update_errors_fn, update_data_profile_table_fn): """ Create delete, impute_x, and impute_y preview tables for a 2D (two-column) selection. Returns a dict with preview table names and dims=2. @@ -541,9 +543,13 @@ def create_previews_2d(table, row_ids, cols, preview_name_fn, update_errors_fn): query.impute_by_ids(table=preview_impute_x, col=cols[0], ids=row_ids) query.impute_by_ids(table=preview_impute_y, col=cols[1], ids=row_ids) - update_errors_fn(preview_delete) - update_errors_fn(preview_impute_x) - update_errors_fn(preview_impute_y) + errors_df_delete = update_errors_fn(preview_delete) + errors_df_impute_x = update_errors_fn(preview_impute_x) + errors_df_impute_y = update_errors_fn(preview_impute_y) + update_data_profile_table_fn(preview_delete, errors_df_delete) + update_data_profile_table_fn(preview_impute_x, errors_df_impute_x) + update_data_profile_table_fn(preview_impute_y, errors_df_impute_y) + return { "success": True, From 3f31ee5f16d98579e1bc889b3bcc45538f105397 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 29 Jun 2026 13:21:46 -0600 Subject: [PATCH 09/81] Added import for DataProfile --- app/server_utils/service_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index b995976..9a04b9a 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -19,6 +19,7 @@ from detectors.datatype_mismatch import datatype_mismatch from detectors.incomplete import incomplete from detectors.missing_value import missing_value +from app.ai_utils.data_attributes import DataProfile def get_current_pgraph(): """ From c1846dce2654dba56f0053dbe3de64008d9982e6 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 29 Jun 2026 13:22:28 -0600 Subject: [PATCH 10/81] Created DataProfile class (still a WIP) --- app/ai_utils/data_attributes.py | 96 ++++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 20 deletions(-) diff --git a/app/ai_utils/data_attributes.py b/app/ai_utils/data_attributes.py index f72c914..ba98064 100644 --- a/app/ai_utils/data_attributes.py +++ b/app/ai_utils/data_attributes.py @@ -2,7 +2,8 @@ import pandas as pd import json -from app.server_utils.service_helpers import is_categorical, get_error_dist +from pandas.core.arrays import categorical + from app.db_utils.execute_sql import fetch_sql @@ -19,23 +20,41 @@ def load_table_to_df(table_name, engine): return df +def to_scalar(val): + if val is None: + return None + if isinstance(val, pd.DataFrame): + if val.empty: + return None + return to_scalar(val.iloc[0, 0]) # first row, first column + if isinstance(val, pd.Series): + return None if val.empty else to_scalar(val.iloc[0]) + if isinstance(val, (list, tuple)) and len(val) > 0: + return to_scalar(val[0]) + if hasattr(val, 'item'): + return val.item() + return val class DataProfile: # TODO: add a way to figure out where the changes were made (more efficient updating of data attributes def __init__(self, table_name, engine=None, main_df=None, error_df=None): self.table_name = table_name + self.data_profile_table_name = "data_profile_" + table_name self.engine = engine self.main_df = main_df self.error_df = error_df + self.default_attributes = ['mean', 'median', 'min', 'max', 'n_categories', 'mode'] self.name_to_func = { 'mean': self.calculate_mean, 'median': self.calculate_median, 'min': self.calculate_min, - 'max': self.calculate_max #TODO: add more + 'max': self.calculate_max, #TODO: add more, + 'n_categories': self.calculate_num_categories, + 'mode': self.calculate_mode } self.attribute_type_assignment = { # TODO: fix this name it sucks lol - 'numeric': ['mean', 'median', 'min', 'max'], - 'categorical': ['n_categories' + 'numeric': ['mean', 'median', 'min', 'max', 'mode'], + 'categorical': ['n_categories', 'mode'] } @@ -48,27 +67,56 @@ def __init__(self, table_name, engine=None, main_df=None, error_df=None): assert engine is not None, f"engine cannot be None if error_df is None" self.error_df = load_table_to_df(f"errors_{table_name}", engine) + + def get_col_data(self, column_name, attribute_name): + assert (attribute_name in self.attribute_type_assignment['categorical'] or attribute_name in self.attribute_type_assignment['numeric']), f"Invalid attribute name {attribute_name}" + + if attribute_name in self.attribute_type_assignment['categorical']: + col_data = self.main_df[column_name].fillna('N/A') + if attribute_name in self.attribute_type_assignment['numeric']: + col_data = pd.to_numeric(self.main_df[column_name], errors='coerce').dropna() + + return col_data + + + def look_up_stat_from_profile(self, attribute_name, column_name): + data_profile_table_name = "dp_" + self.table_name + try: + query = f'SELECT {attribute_name} FROM "{data_profile_table_name}" WHERE column_name = "{column_name}"' + stat = fetch_sql(query, False, self.engine) + + # TODO: make sure that when nothing matches the query, this still will work + if stat is None: + assert False, "AHHHHHHHHHHHHHHHHHHHHHHHHHHHH THIS SHOULDNT BE HAPPENING!!!!!!!!!" + elif stat.empty: # TODO: idk what I'm doing here + return None + return stat + except Exception as e: + print(f"Attribute {attribute_name} not found from data profile.") + def query_summary_stat_from_main_df(self, stat_query, column_name): # TODO: fix this probably - query = f'SELECT {stat_query}({column_name}) FROM "{self.table_name}"' - stat = fetch_sql(query, False, self.engine) + query = f'SELECT {stat_query}("{column_name}") FROM "{self.table_name}"' + stat = fetch_sql(query, True, self.engine) return stat - def calculate_column_attribute(self, attribute_name, column_name): - look_up_value = self.look_up_stat(self, attribute_name, column_name) - # TODO: add asserts to make sure that attribute is compatible with the type of the column - if look_up_value is not None: - return look_up_value + def calculate_column_attribute(self, attribute_name, column_name, look_up_stat=True): + if look_up_stat: + look_up_value = self.look_up_stat_from_profile(attribute_name, column_name) + + if look_up_value is not None: + return to_scalar(look_up_value) calculate_attribute_func = self.name_to_func[attribute_name] - return calculate_attribute_func(column_name) + return to_scalar(calculate_attribute_func(column_name)) # TODO: save stat off to table if newly calculated def calculate_mean(self, column_name): + col_data = self.get_col_data(column_name, 'mean') try: avg = self.query_summary_stat_from_main_df('AVG', column_name) except Exception as e: @@ -76,13 +124,15 @@ def calculate_mean(self, column_name): print("Calculating mean manually using data...") - avg = self.main_df[column_name].mean() + avg = col_data.mean() print(f"Updating mean value for column {column_name} in data profile") return avg def calculate_median(self, column_name): + col_data = self.get_col_data(column_name, 'median') + try: median = self.query_summary_stat_from_main_df('MEDIAN', column_name) except Exception as e: @@ -90,7 +140,7 @@ def calculate_median(self, column_name): print("Calculating median manually using data...") - median = self.main_df[column_name].median() + median = col_data.median() print(f"Updating median value for column {column_name} in data profile") @@ -98,6 +148,7 @@ def calculate_median(self, column_name): def calculate_max(self, column_name): + col_data = self.get_col_data(column_name, 'max') try: maximum = self.query_summary_stat_from_main_df('MAX', column_name) except Exception as e: @@ -105,12 +156,15 @@ def calculate_max(self, column_name): print("Calculating maximum manually using data...") - maximum = self.main_df[column_name].max() + maximum = col_data.max() print(f"Updating maximum value for column {column_name} in data profile") + return maximum def calculate_min(self, column_name): + + col_data = self.get_col_data(column_name, 'min') try: minimum = self.query_summary_stat_from_main_df('MIN', column_name) except Exception as e: @@ -118,22 +172,23 @@ def calculate_min(self, column_name): print("Calculating minimum manually using data...") - minimum = self.main_df[column_name].min() + minimum = col_data.min() print(f"Updating minimum value for column {column_name} in data profile") return minimum def calculate_num_categories(self, column_name): + col_data = self.get_col_data(column_name, 'n_categories') try: query = f'SELECT COUNT(DISTINCT "{column_name}") FROM "{self.table_name}"' - n_categories = fetch_sql(query, False, self.engine) + n_categories = fetch_sql(query, True, self.engine) except Exception as e: print(f"Error fetching the n_categories for table {self.table_name} at column {column_name}: {e}") print("Calculating n_categories manually using data...") - n_categories = self.main_df[column_name].nunique() + n_categories = col_data.nunique() print(f"Updating n_categories value for column {column_name} in data profile") @@ -142,11 +197,12 @@ def calculate_num_categories(self, column_name): def calculate_mode(self, column_name): print("Calculating mode manually using data...") - mode = self.main_df[column_name].mode() + col_data = self.get_col_data(column_name, 'mode') + mode = col_data.mode() return mode - + # TODO: calculate some stats relating to the error df From 4388c4a7534ee3b06ed587e13354eb62330feaf3 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 11:59:10 -0600 Subject: [PATCH 11/81] Switched functions from pandas df summary stats functions to (slightly) optimized functions from the DataProfile class. --- .../data_attribute_summary_integration.py | 57 +++++++++---------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/app/server_utils/data_attribute_summary_integration.py b/app/server_utils/data_attribute_summary_integration.py index c68a254..b41af09 100644 --- a/app/server_utils/data_attribute_summary_integration.py +++ b/app/server_utils/data_attribute_summary_integration.py @@ -9,6 +9,7 @@ from app.db_utils.execute_sql import fetch_sql from app.server_utils.service_helpers import get_error_dist, is_categorical, _validate_identifier +from app.db_utils.data_attributes import DataProfile def get_default_attributes_from_rankings(tablename, engine): @@ -43,75 +44,71 @@ def generate_complete_json(tablename): _validate_identifier(tablename) print(f"Generating JSON for table: {tablename}") - main_rows = fetch_sql(f'SELECT * FROM "{tablename}"', False, engine) - error_rows = fetch_sql(f'SELECT * FROM "errors_{tablename}"', False, engine) - - main_cols = fetch_sql(f"SELECT column_name FROM information_schema.columns WHERE table_name = '{tablename}' ORDER BY ordinal_position", False, engine) - error_cols = fetch_sql(f"SELECT column_name FROM information_schema.columns WHERE table_name = 'errors_{tablename}' ORDER BY ordinal_position", False, engine) - - main_df = pd.DataFrame(main_rows, columns=[row[0] for row in main_cols] if main_cols else None) - error_df = pd.DataFrame(error_rows, columns=[row[0] for row in error_cols] if error_cols else None) + data_profile = DataProfile(tablename, engine) + error_df = data_profile.get_error_df() + main_df = data_profile.get_main_df() error_list = get_error_dist(error_df, main_df).to_dict('records') + default_attributes = get_default_attributes_from_rankings(tablename, engine) + return { "columnErrors": convert_error_list_to_dict(error_list), - "attributes": list(main_df.columns), - "attributeDistributions": build_attribute_distributions(main_df), + "attributes": list(data_profile._main_df.columns), + "attributeDistributions": build_attribute_distributions(data_profile), "defaultAttributes": default_attributes } -def get_attribute_stats(df, column): +def get_attribute_stats(data_profile, column): """ Get statistics for a specific attribute in the DataFrame - :param df: DataFrame containing the data + :param data_profile: Data profile class instance (used for calculating summary stats) :param column: name of the column to get statistics for :return: dictionary containing statistics for the column """ - if is_categorical(df[column]): - return get_categorical_stats(df, column) - return get_numeric_stats(df, column) + if is_categorical(data_profile._main_df[column]): + return get_categorical_stats(data_profile, column) + return get_numeric_stats(data_profile, column) -def build_attribute_distributions(main_df): +def build_attribute_distributions(data_profile): """ Build distributions for each attribute in the main DataFrame - :param main_df: DataFrame containing the main data + :param data_profile: Data profile class instance (used for calculating summary stats) :return: dictionary containing distributions for each attribute """ distributions = {} - for col in main_df.columns: - distributions[col] = get_attribute_stats(main_df, col) + + for col in data_profile.get_col_names(): + distributions[col] = get_attribute_stats(data_profile, col) return distributions -def get_categorical_stats(df, column): +def get_categorical_stats(data_profile, column): """ Get statistics for a categorical attribute in the DataFrame - :param df: DataFrame containing the data + :param data_profile: DataProfile class instance with optimized summary stat calculation functions :param column: name of the column to get statistics for :return: dictionary containing statistics for the categorical column """ - col_data = df[column].fillna('N/A') return { "categorical": { - "categories": col_data.nunique(), - "mode": col_data.mode().iloc[0] + "categories": data_profile.calculate_column_attribute('n_categories', column), + "mode": data_profile.calculate_column_attribute('mode', column) } } -def get_numeric_stats(df, column): +def get_numeric_stats(data_profile, column): """ Get statistics for a numeric attribute in the DataFrame - :param df: DataFrame containing the data + :param data_profile: DataProfile class instance with optimized summary stat calculation functions :param column: name of the column to get statistics for :return: dictionary containing statistics for the numeric column """ - col_data = pd.to_numeric(df[column], errors='coerce').dropna() return { "numeric": { - "mean": col_data.mean().item(), - "min": col_data.min().item(), - "max": col_data.max().item() + "mean": data_profile.calculate_column_attribute('mean', column), + "min": data_profile.calculate_column_attribute('min', column), + "max": data_profile.calculate_column_attribute('max', column) } } From 0f7fa11117573fdb2af490b53d9c101b9a5b2af6 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 11:59:34 -0600 Subject: [PATCH 12/81] Removed pandas import --- app/server_utils/data_attribute_summary_integration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/server_utils/data_attribute_summary_integration.py b/app/server_utils/data_attribute_summary_integration.py index b41af09..e34b9c5 100644 --- a/app/server_utils/data_attribute_summary_integration.py +++ b/app/server_utils/data_attribute_summary_integration.py @@ -5,7 +5,6 @@ """ Refactored to use jacobs new backend sql files March 11,2026 - db_functions_sql.py, execute_sql.py, filtering_sql.py """ -import pandas as pd from app.db_utils.execute_sql import fetch_sql from app.server_utils.service_helpers import get_error_dist, is_categorical, _validate_identifier From c04878e2c3c7f23e4784c27b0b60bc93c26bac05 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 11:59:58 -0600 Subject: [PATCH 13/81] Moved data_attributes.py from ai_utils to db_utils --- app/ai_utils/data_attributes.py | 306 -------------------------------- 1 file changed, 306 deletions(-) delete mode 100644 app/ai_utils/data_attributes.py diff --git a/app/ai_utils/data_attributes.py b/app/ai_utils/data_attributes.py deleted file mode 100644 index ba98064..0000000 --- a/app/ai_utils/data_attributes.py +++ /dev/null @@ -1,306 +0,0 @@ -import numpy as np -import pandas as pd -import json - -from pandas.core.arrays import categorical - -from app.db_utils.execute_sql import fetch_sql - - - -# TODO: change name to data profile instead of ai_utils / related -# TODO: Make this work with this with server_utils/data_attribute_summary_integration.py to create a data profile class that handles this shit -# This one would just be data profile class while ^^^^ uses it to get the data attribute summaries to show stuff - -# maybe move this to a utils script or something? -def load_table_to_df(table_name, engine): - rows = fetch_sql(f'SELECT * FROM "{table_name}"', False, engine) - cols = fetch_sql(f"SELECT column_name FROM information_schema.columns WHERE table_name = '{table_name}' ORDER BY ordinal_position", False, engine) - df = pd.DataFrame(rows, columns=[row[0] for row in cols] if cols else None) - - return df - -def to_scalar(val): - if val is None: - return None - if isinstance(val, pd.DataFrame): - if val.empty: - return None - return to_scalar(val.iloc[0, 0]) # first row, first column - if isinstance(val, pd.Series): - return None if val.empty else to_scalar(val.iloc[0]) - if isinstance(val, (list, tuple)) and len(val) > 0: - return to_scalar(val[0]) - if hasattr(val, 'item'): - return val.item() - return val - -class DataProfile: - # TODO: add a way to figure out where the changes were made (more efficient updating of data attributes - def __init__(self, table_name, engine=None, main_df=None, error_df=None): - self.table_name = table_name - self.data_profile_table_name = "data_profile_" + table_name - self.engine = engine - self.main_df = main_df - self.error_df = error_df - self.default_attributes = ['mean', 'median', 'min', 'max', 'n_categories', 'mode'] - self.name_to_func = { - 'mean': self.calculate_mean, - 'median': self.calculate_median, - 'min': self.calculate_min, - 'max': self.calculate_max, #TODO: add more, - 'n_categories': self.calculate_num_categories, - 'mode': self.calculate_mode - } - self.attribute_type_assignment = { # TODO: fix this name it sucks lol - 'numeric': ['mean', 'median', 'min', 'max', 'mode'], - 'categorical': ['n_categories', - 'mode'] - } - - - if self.main_df is None: - assert engine is not None, f"engine cannot be None if main_df is None" - self.main_df = load_table_to_df(table_name, engine) - - if self.error_df is None: - assert engine is not None, f"engine cannot be None if error_df is None" - self.error_df = load_table_to_df(f"errors_{table_name}", engine) - - - def get_col_data(self, column_name, attribute_name): - assert (attribute_name in self.attribute_type_assignment['categorical'] or attribute_name in self.attribute_type_assignment['numeric']), f"Invalid attribute name {attribute_name}" - - if attribute_name in self.attribute_type_assignment['categorical']: - col_data = self.main_df[column_name].fillna('N/A') - if attribute_name in self.attribute_type_assignment['numeric']: - col_data = pd.to_numeric(self.main_df[column_name], errors='coerce').dropna() - - return col_data - - - - def look_up_stat_from_profile(self, attribute_name, column_name): - data_profile_table_name = "dp_" + self.table_name - try: - query = f'SELECT {attribute_name} FROM "{data_profile_table_name}" WHERE column_name = "{column_name}"' - stat = fetch_sql(query, False, self.engine) - - # TODO: make sure that when nothing matches the query, this still will work - if stat is None: - assert False, "AHHHHHHHHHHHHHHHHHHHHHHHHHHHH THIS SHOULDNT BE HAPPENING!!!!!!!!!" - elif stat.empty: # TODO: idk what I'm doing here - return None - return stat - except Exception as e: - print(f"Attribute {attribute_name} not found from data profile.") - - - - def query_summary_stat_from_main_df(self, stat_query, column_name): - # TODO: fix this probably - query = f'SELECT {stat_query}("{column_name}") FROM "{self.table_name}"' - stat = fetch_sql(query, True, self.engine) - return stat - - - def calculate_column_attribute(self, attribute_name, column_name, look_up_stat=True): - if look_up_stat: - look_up_value = self.look_up_stat_from_profile(attribute_name, column_name) - - if look_up_value is not None: - return to_scalar(look_up_value) - - calculate_attribute_func = self.name_to_func[attribute_name] - return to_scalar(calculate_attribute_func(column_name)) - - # TODO: save stat off to table if newly calculated - def calculate_mean(self, column_name): - col_data = self.get_col_data(column_name, 'mean') - try: - avg = self.query_summary_stat_from_main_df('AVG', column_name) - except Exception as e: - print(f"Error fetching the mean for table {self.table_name} at column {column_name}: {e}") - - print("Calculating mean manually using data...") - - avg = col_data.mean() - print(f"Updating mean value for column {column_name} in data profile") - - return avg - - - def calculate_median(self, column_name): - col_data = self.get_col_data(column_name, 'median') - - try: - median = self.query_summary_stat_from_main_df('MEDIAN', column_name) - except Exception as e: - print(f"Error fetching the median for table {self.table_name} at column {column_name}: {e}") - - print("Calculating median manually using data...") - - median = col_data.median() - - print(f"Updating median value for column {column_name} in data profile") - - return median - - - def calculate_max(self, column_name): - col_data = self.get_col_data(column_name, 'max') - try: - maximum = self.query_summary_stat_from_main_df('MAX', column_name) - except Exception as e: - print(f"Error fetching the maximum for table {self.table_name} at column {column_name}: {e}") - - print("Calculating maximum manually using data...") - - maximum = col_data.max() - - print(f"Updating maximum value for column {column_name} in data profile") - - return maximum - - def calculate_min(self, column_name): - - col_data = self.get_col_data(column_name, 'min') - try: - minimum = self.query_summary_stat_from_main_df('MIN', column_name) - except Exception as e: - print(f"Error fetching the minimum for table {self.table_name} at column {column_name}: {e}") - - print("Calculating minimum manually using data...") - - minimum = col_data.min() - - print(f"Updating minimum value for column {column_name} in data profile") - - return minimum - - def calculate_num_categories(self, column_name): - col_data = self.get_col_data(column_name, 'n_categories') - try: - query = f'SELECT COUNT(DISTINCT "{column_name}") FROM "{self.table_name}"' - n_categories = fetch_sql(query, True, self.engine) - except Exception as e: - print(f"Error fetching the n_categories for table {self.table_name} at column {column_name}: {e}") - - print("Calculating n_categories manually using data...") - - n_categories = col_data.nunique() - - print(f"Updating n_categories value for column {column_name} in data profile") - - return n_categories - - - def calculate_mode(self, column_name): - print("Calculating mode manually using data...") - col_data = self.get_col_data(column_name, 'mode') - mode = col_data.mode() - - return mode - - # TODO: calculate some stats relating to the error df - - - - - - - - - - - - - - - - - -''' - - - -# Gets the data attributes that are given to LLM -def get_data_attributes(tablename): - from app import engine - - # TODO: already a similar loop in data_attribute_summary_integration.py - # line 85, maybe somehow combine these? Idk if it makes sense to put ai - # related things in there though? - # build_attribute_distributions() very similar; combine into same function in future - main_rows = fetch_sql(f'SELECT * FROM "{tablename}"', False, engine) - - main_cols = fetch_sql( - f"SELECT column_name FROM information_schema.columns WHERE table_name = '{tablename}' ORDER BY ordinal_position", - False, engine) - - main_df = pd.DataFrame(main_rows, columns=[row[0] for row in main_cols] if main_cols else None) - - data_attributes = {} - for col in main_df.columns: - data_attributes[col] = get_col_attributes(main_df, col) - - return data_attributes - -def save_data_attributes(tablename, json_path="data_attributes.json"): - data_attributes = get_data_attributes(tablename) - with open(f'app/ai_utils/{json_path}', 'w') as outfile: - json.dump(data_attributes, outfile) - -# TODO: maybe use scipy so it's more efficient? -def get_median_absolute_deviation(col_data, median): - median_residuals = col_data - median - abs_median_residuals = np.abs(median_residuals) - mad = int(np.median(abs_median_residuals)) - return mad - -# Returns a list: [lower_fence, upper_fence] -def get_tukeys_fences(col_data, iqr): - q1 = np.percentile(col_data, 25) - q3 = np.percentile(col_data, 75) - lower_fence = (q1 - 1.5 * (iqr)).item() - upper_fence = (q3 + 1.5 * (iqr)).item() - - return (lower_fence, upper_fence) - -def get_col_attributes(df, col): - col_attributes = {} - col_data = df[col] - - col_attributes['data_type'] = str(col_data.dtype) - col_attributes['num_na'] = int(col_data.isnull().sum()) - col_attributes['count'] = int(col_data.count()) - - if is_categorical(col_data): - col_data = df[col].fillna('N/A') - col_attributes['num_unique'] = int(col_data.nunique()) - # TODO: maybe do proportion instead? - col_attributes['category_count'] = col_data.value_counts().to_dict() - else: - col_data = pd.to_numeric(df[col], errors='coerce').dropna() - col_attributes['mean'] = col_data.mean().item() - col_attributes['median'] = col_data.median().item() - col_attributes['min'] = col_data.min().item() - col_attributes['max'] = col_data.max().item() - # ik a lot of these do the same things but idk which one to choose - # --- measures of dispersion - col_attributes['var'] = col_data.var().item() - col_attributes['iqr'] = (col_data.quantile(0.75) - col_data.quantile(0.25)).item() - col_attributes['std'] = col_data.std().item() - col_attributes['skew'] = col_data.skew().item() - col_attributes['median_absolute_deviation'] = get_median_absolute_deviation(col_data, col_attributes['median']) - # ------- - - # List where first val is lower fence, second is upper fence - # Can't be tuple bc jsons don't support tuples - col_attributes['tukeys_fence'] = get_tukeys_fences(col_data, col_attributes['iqr']) - return col_attributes - - - - -''' \ No newline at end of file From 8fc8090b2f4f36a64fb610f55abe2df4b7c1bf4a Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 12:00:03 -0600 Subject: [PATCH 14/81] Moved data_attributes.py from ai_utils to db_utils --- app/db_utils/data_attributes.py | 282 ++++++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 app/db_utils/data_attributes.py diff --git a/app/db_utils/data_attributes.py b/app/db_utils/data_attributes.py new file mode 100644 index 0000000..f2b8b2a --- /dev/null +++ b/app/db_utils/data_attributes.py @@ -0,0 +1,282 @@ +import numpy as np +import pandas as pd +import json + +from pandas.core.arrays import categorical + +from app.db_utils.execute_sql import fetch_sql + + + +#TODO: add documentation + +def to_scalar(val): + if val is None: + return None + if isinstance(val, pd.DataFrame): + if val.empty: + return None + return to_scalar(val.iloc[0, 0]) # first row, first column + if isinstance(val, pd.Series): + return None if val.empty else to_scalar(val.iloc[0]) + if isinstance(val, (list, tuple)) and len(val) > 0: + return to_scalar(val[0]) + if hasattr(val, 'item'): + return val.item() + return val + +class DataProfile: + def __init__(self, table_name, engine=None, main_df=None, error_df=None): + self.table_name = table_name + self.data_profile_table_name = "dp" + table_name + self.engine = engine + # IMPORTANT: main_df and error_df are NOT guaranteed to not be None (so that they don't have to be loaded each time for efficiency) + # Use get_main_df and get_error_df instead of accessing them directly + self._main_df = main_df + self._error_df = error_df + self.default_attributes = ['mean', 'median', 'min', 'max', 'n_categories', 'mode', 'error_counts', 'class_error_counts'] + self.name_to_func = { + 'mean': self._calculate_mean, + 'median': self._calculate_median, + 'min': self._calculate_min, + 'max': self._calculate_max, #TODO: add more, + 'n_categories': self._calculate_num_categories, + 'mode': self._calculate_mode, + 'error_counts': self._calculate_error_count_dict, + 'class_error_counts': self._calculate_class_error_count_dict, + } + + self.attribute_type_assignment = { + 'numeric': ['mean', 'median', 'min', 'max', 'err', 'error_counts'], + 'categorical': ['n_categories', + 'mode', 'error_counts', 'class_error_counts'], + } + + def get_error_df(self): + self.load_error_df() + + return self._error_df + + def get_main_df(self): + self.load_main_df() + + return self._main_df + + + def load_error_df(self): + if self._error_df is None: + assert self.engine is not None, f"engine cannot be None if error_df is None" + #self.error_df = load_table_to_df(f"errors_{self.table_name}", self.engine) + self._error_df = pd.read_sql_query(f'SELECT * FROM "{"errors_" + self.table_name}"', self.engine) + + def load_main_df(self): + if self._main_df is None: + assert self.engine is not None, f"engine cannot be None if main_df is None" + #self.main_df = load_table_to_df(self.table_name, self.engine) + self._main_df = pd.read_sql_query(f'SELECT * FROM "{self.table_name}"', self.engine) + + + + def get_col_data(self, column_name, attribute_name): + assert (attribute_name in self.attribute_type_assignment['categorical'] or attribute_name in self.attribute_type_assignment['numeric']), f"Invalid attribute name {attribute_name}" + + if attribute_name in self.attribute_type_assignment['categorical']: + col_data = self._main_df[column_name].fillna('N/A') + if attribute_name in self.attribute_type_assignment['numeric']: + col_data = pd.to_numeric(self._main_df[column_name], errors='coerce').dropna() + + return col_data + + # TODO: Make the sql query for this work + def get_col_names(self): + try: + query = 'SELECT column_name FROM information_schema.columns WHERE table_name = :table_name ORDER BY ordinal_position' + params = {"table_name": self.table_name} + + result = fetch_sql(query, False, self.engine, params=params) + + col_names = [] + for row in result: + if row[0] not in ['index', 'level_0', ]: + col_names.append(row[0]) + + print("COL NAMES FROM QUERY: ", col_names) + + + except Exception as e: + print(f"AHHHHHHHHHH Querying for col names unsuccessful because of error: {e}") + print("Getting col names from Data frame instead") + self.load_main_df() + print("COL NAMES FROM main_df.columns:", self._main_df.columns) + + col_names = self._main_df.columns + + return col_names + + def look_up_stat_from_profile(self, attribute_name, column_name): + data_profile_table_name = "dp_" + self.table_name + try: + query = f'SELECT "{attribute_name}" FROM "{data_profile_table_name}" WHERE "column_name" = :column_name' + params = {"column_name": column_name} + stat = fetch_sql(query, True, self.engine, params) + + return stat + except Exception as e: + print(f"Error querying attribute from data profile table: {e}") + + def calculate_summary_stat_using_sql(self, stat_query, column_name): + query = f'SELECT {stat_query}("{column_name}") FROM "{self.table_name}"' + stat = fetch_sql(query, True, self.engine) + return stat + + def calculate_column_attribute(self, attribute_name, column_name, look_up_stat=True): + if look_up_stat: + look_up_value = self.look_up_stat_from_profile(attribute_name, column_name) + + if look_up_value is not None: + print("Successfully found col attribute in data profile table!") + return to_scalar(look_up_value) + + calculate_attribute_func = self.name_to_func[attribute_name] + return to_scalar(calculate_attribute_func(column_name)) + + # TODO: save stat off to table if newly calculated + def _calculate_mean(self, column_name): + col_data = self.get_col_data(column_name, 'mean') + try: + avg = self.calculate_summary_stat_using_sql('AVG', column_name) + except Exception as e: + print(f"Error fetching the mean for table {self.table_name} at column {column_name}: {e}") + + print("Calculating mean manually using data...") + self.load_main_df() + + avg = col_data.mean() + print(f"Updating mean value for column {column_name} in data profile") + + return avg + + + def _calculate_median(self, column_name): + col_data = self.get_col_data(column_name, 'median') + + try: + query = f'SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY "{column_name}") FROM "{self.table_name}"' + median = fetch_sql(query, True, self.engine) + except Exception as e: + self.load_main_df() + print(f"Error fetching the median for table {self.table_name} at column {column_name}: {e}") + + print("Calculating median manually using data...") + + median = col_data.median() + + print(f"Updating median value for column {column_name} in data profile") + + return median + + + def _calculate_max(self, column_name): + col_data = self.get_col_data(column_name, 'max') + try: + maximum = self.calculate_summary_stat_using_sql('MAX', column_name) + except Exception as e: + print(f"Error fetching the maximum for table {self.table_name} at column {column_name}: {e}") + + print("Calculating maximum manually using data...") + self.load_main_df() + + maximum = col_data.max() + + print(f"Updating maximum value for column {column_name} in data profile") + + return maximum + + def _calculate_min(self, column_name): + + col_data = self.get_col_data(column_name, 'min') + try: + minimum = self.calculate_summary_stat_using_sql('MIN', column_name) + except Exception as e: + print(f"Error fetching the minimum for table {self.table_name} at column {column_name}: {e}") + + print("Calculating minimum manually using data...") + self.load_main_df() + + minimum = col_data.min() + + print(f"Updating minimum value for column {column_name} in data profile") + + return minimum + + def _calculate_num_categories(self, column_name): + col_data = self.get_col_data(column_name, 'n_categories') + try: + query = f'SELECT COUNT(DISTINCT "{column_name}") FROM "{self.table_name}"' + n_categories = fetch_sql(query, True, self.engine) + + except Exception as e: + print("AHHH SQL QUERY DIDN'T WORK") + print(f"Error fetching the n_categories for table {self.table_name} at column {column_name}: {e}") + + print("Calculating n_categories manually using data...") + self.load_main_df() + + n_categories = col_data.nunique() + + print(f"Updating n_categories value for column {column_name} in data profile") + + return n_categories + + + def _calculate_mode(self, column_name): + print("Calculating mode manually using data...") + self.load_main_df() + col_data = self.get_col_data(column_name, 'mode') + mode = col_data.mode() + + return mode + + # Functions relating to error info + # Dict mapping from error type to total error count + def _calculate_error_count_dict(self, column_name): + try: + query = f'SELECT {column_name}, COUNT(*) as cnt FROM {self.table_name} GROUP BY {column_name} ORDER BY cnt DESC' + category_counts = dict(fetch_sql(query, True, self.engine)) + except Exception as e: + + print(f"Error fetching the error counts for table {self.table_name} at column {column_name}: {e}") + + print("Calculating error counts manually using data...") + self.load_error_df() + + category_counts = {} + if not self._error_df.empty: # If error_df is empty (no errors in data selection) + category_counts = self._error_df["error_type"].value_counts().to_dict() + + if category_counts is not None: + category_counts = json.dumps(category_counts) + + return category_counts + + # Dict mapping from class to error types to error counts + # TODO: Implement SQL query version + def _calculate_class_error_count_dict(self, column_name): + print("Calculating class error counts manually using data...") + + self.load_error_df() + + counts_by_column = {} + if not self._error_df.empty: # If error_df is empty (no errors in data selection) + counts_by_column = ( + self._error_df.groupby(['column_id', 'error_type']) + .size() + .unstack(fill_value=0) + .to_dict(orient='index') + ) + + if counts_by_column is not None: + counts_by_column = json.dumps(counts_by_column) + + + return counts_by_column From 812c5e5ea129746a72fa7970d8bb501c9645c3a4 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 12:00:39 -0600 Subject: [PATCH 15/81] Renamed data_attributes.py to data_profile.py --- app/db_utils/{data_attributes.py => data_profile.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename app/db_utils/{data_attributes.py => data_profile.py} (100%) diff --git a/app/db_utils/data_attributes.py b/app/db_utils/data_profile.py similarity index 100% rename from app/db_utils/data_attributes.py rename to app/db_utils/data_profile.py From 8bef2a962777ffe144b6246bf746548bb2ecc0f7 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 12:01:08 -0600 Subject: [PATCH 16/81] Renamed data_attributes.py to data_profile.py --- app/server_utils/data_attribute_summary_integration.py | 2 +- app/server_utils/service_helpers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/server_utils/data_attribute_summary_integration.py b/app/server_utils/data_attribute_summary_integration.py index e34b9c5..402a6ff 100644 --- a/app/server_utils/data_attribute_summary_integration.py +++ b/app/server_utils/data_attribute_summary_integration.py @@ -8,7 +8,7 @@ from app.db_utils.execute_sql import fetch_sql from app.server_utils.service_helpers import get_error_dist, is_categorical, _validate_identifier -from app.db_utils.data_attributes import DataProfile +from app.db_utils.data_profile import DataProfile def get_default_attributes_from_rankings(tablename, engine): diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index 9a04b9a..1eb840d 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -19,7 +19,7 @@ from detectors.datatype_mismatch import datatype_mismatch from detectors.incomplete import incomplete from detectors.missing_value import missing_value -from app.ai_utils.data_attributes import DataProfile +from app.db_utils.data_profile import DataProfile def get_current_pgraph(): """ From 737c2ac83bb90d22b7797cb0fb411e98bbd9a2f3 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 12:24:49 -0600 Subject: [PATCH 17/81] Added data profile table to the preview table functions --- app/db_utils/db_functions_sql.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/db_utils/db_functions_sql.py b/app/db_utils/db_functions_sql.py index a57ad0d..86cc55f 100644 --- a/app/db_utils/db_functions_sql.py +++ b/app/db_utils/db_functions_sql.py @@ -183,6 +183,7 @@ def drop_preview_tables(self, all_possible_previews: list, keep_table: str): if pt != keep_table: execute_sql(f'DROP TABLE IF EXISTS "{pt}"', self.engine) execute_sql(f'DROP TABLE IF EXISTS "errors_{pt}"', self.engine) + execute_sql(f'DROP TABLE IF EXISTS "dp_{pt}"', self.engine) def rename_preview_to_new(self, preview_table: str, new_table_name: str): """ @@ -192,6 +193,7 @@ def rename_preview_to_new(self, preview_table: str, new_table_name: str): # self.engine.dispose() execute_sql(f'ALTER TABLE "{preview_table}" RENAME TO "{new_table_name}"', self.engine) execute_sql(f'ALTER TABLE IF EXISTS "errors_{preview_table}" RENAME TO "errors_{new_table_name}"', self.engine) + execute_sql(f'ALTER TABLE IF EXISTS "dp_{preview_table}" RENAME TO "dp_{new_table_name}"', self.engine) def remove_data_filters(self, sql_filters) -> dict: From a3055a2590d3a4dc2e4fbd57ef583614b8768bc3 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 12:26:08 -0600 Subject: [PATCH 18/81] Added creation of data profile table into load_file() --- app/routes/routes.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/routes/routes.py b/app/routes/routes.py index 5bd2b07..53384cf 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -11,8 +11,7 @@ generate_table_name, run_detectors, get_sqlalchemy_dtype_map, - calculate_attribute_rankings, get_pgraph_redo, get_pgraph_undo, init_pgraph_for_session, - + calculate_attribute_rankings, get_pgraph_redo, get_pgraph_undo, init_pgraph_for_session, create_data_profile_df, ) from app.server_utils.set_id_column import set_id_column @@ -52,8 +51,13 @@ def load_file(csv_file, filename): The number of returned rows affected is the sum of the rowcount attribute of sqlite3.Cursor or SQLAlchemy connectable which may not reflect the exact number of written rows as stipulated in the sqlite3 or SQLAlchemy. """ - rows_affected = table_with_id_added.to_sql(table_name_with_node_id, engine, if_exists='replace', dtype=dtype_map) - detected_rows_affected = detected_data.to_sql("errors_" + table_name_with_node_id, engine, if_exists='replace') + table_with_id_added.to_sql(table_name_with_node_id, engine, if_exists='replace', dtype=dtype_map) + detected_data.to_sql("errors_" + table_name_with_node_id, engine, if_exists='replace') + + data_profile_df = create_data_profile_df(table_name_with_node_id, engine, error_df=detected_data, main_df=dataframe) + + data_profile_df.to_sql("dp_" + table_name_with_node_id, engine, if_exists='replace') + """ now we fully init the DBOperations object that was first initialized in init.py, From adfcc71c5d04a5df032e7e140145b653608ece1d Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 12:26:49 -0600 Subject: [PATCH 19/81] Fixed bug in create_data_profile_df --- app/server_utils/service_helpers.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index 1eb840d..a310094 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -176,16 +176,21 @@ def run_detectors(data_frame): frames = [anomaly_df, incomplete_df, missing_value_df,datatype_mismatch_df] return perform_melt(frames) -# TODO: modify this so it only updates the changed columns -def create_data_profile_df(table_name, engine, error_df, columns_to_include=None): +# TODO: CLEAN UP THIS FUNCTION +# TODO: Test this function!!! (Write test for it) +def create_data_profile_df(table_name, engine, col_names=None, error_df=None, main_df=None): + print("CREATED DATA_PROFILE DF FOR TABLE", table_name) - data_profile = DataProfile(table_name, engine=engine, error_df=error_df) - if columns_to_include is None: - columns_to_include = data_profile.main_df.columns + data_profile = DataProfile(table_name, engine=engine, main_df=main_df, error_df=error_df) col_attribute_list = [] - for col in columns_to_include: + if col_names is None: + col_names = data_profile.get_col_names() + + for col in col_names: + if col not in data_profile.get_col_names(): + continue row_dict = {'column_name': col} for attribute in data_profile.default_attributes: @@ -193,11 +198,6 @@ def create_data_profile_df(table_name, engine, error_df, columns_to_include=None # Attribute doesn't exist in either categorical and numeric print(f"ERROR: INVALID ATTRIBUTE {attribute}") print("Skipping this attribute") - elif (is_categorical(data_profile.main_df[col]) and attribute not in data_profile.attribute_type_assignment['categorical']) or \ - (not is_categorical(data_profile.main_df[col]) and attribute not in data_profile.attribute_type_assignment['numeric']): - row_dict[attribute] = None - - # default attribute does is not compatible with column continue print("CALCULATING ATTRIBUTE: ", attribute) From c2f47a6be13ea84eb8d216e924cfeb41152a8d4a Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 12:28:09 -0600 Subject: [PATCH 20/81] Added loading of dp table into _clone_table_pair --- app/server_utils/service_helpers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index a310094..7e9728a 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -427,14 +427,18 @@ def execute_wrangle_preview(table, preview_table, preview_name_fn, db_operations return {"success": True, "table": new_table_name} -def _clone_table_pair(conn, source_table, dest_table, errors_source): - """Drop-and-recreate dest_table and its errors_ sibling as copies of source tables.""" +def _clone_table_pair(conn, source_table, dest_table, errors_source, dp_source): + """Drop-and-recreate dest_table and its errors_ and dp_ sibling as copies of source tables.""" conn.execute(sa_text(f'DROP TABLE IF EXISTS "{dest_table}"')) conn.execute(sa_text(f'CREATE TABLE "{dest_table}" AS SELECT * FROM "{source_table}"')) errors_dest = f"errors_{dest_table}" conn.execute(sa_text(f'DROP TABLE IF EXISTS "{errors_dest}"')) conn.execute(sa_text(f'CREATE TABLE "{errors_dest}" AS SELECT * FROM "{errors_source}"')) + dp_dest = f"dp_{dest_table}" + conn.execute(sa_text(f'DROP TABLE IF EXISTS "{dp_dest}"')) + conn.execute(sa_text(f'CREATE TABLE "{dp_dest}" AS SELECT * FROM "{dp_source}"')) + def trim_preview_suffix(name: str) -> str: """Remove the '_preview...' tail from a table name, if present.""" idx = name.find("_preview") From 2c3f9ce2f5256ec1e8513004cd5092f7e48aaacb Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 12:30:23 -0600 Subject: [PATCH 21/81] Refactored create_previews args 'preview_name_fn' to 'safe_pg_name_fn' --- app/server_utils/service_helpers.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index 7e9728a..fb03807 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -463,6 +463,7 @@ def make_new_table_name(child_table): node_id = app.pgraph_for_session.get_new_node_id() new_table_name = f"{node_id}{child_table[2:]}" return new_table_name + def create_minimal_preview_table(conn, source_table, preview_table_name, errors_source, cols): """ Drop and recreate a minimal dest table and empty error table to populate only with regard to the cols. @@ -486,16 +487,18 @@ def create_minimal_preview_table(conn, source_table, preview_table_name, errors_ # Copy schema but leave empty. conn.execute(sa_text(f'CREATE TABLE "{errors_dest}" (LIKE "{errors_source}" INCLUDING ALL)"')) -def create_previews_1d(table, row_ids, cols, preview_name_fn, update_errors_fn, update_data_profile_table_fn): +def create_previews_1d(table, row_ids, cols, safe_pg_name_fn, update_errors_fn, update_data_profile_table_fn): """ Create delete and impute preview tables for a 1D (single-column) selection. Returns a dict with preview table names and dims=1. """ from app import engine - errors_src = f"errors_{table}" - preview_delete = preview_name_fn(table, "_preview_delete") - preview_impute = preview_name_fn(table, "_preview_impute") + errors_src = f"errors_{table}" + dp_src = f"dp_{table}" + # Creates name for the preview tables + preview_delete_table_name = safe_pg_name_fn(table, "_preview_delete") + preview_impute_table_name = safe_pg_name_fn(table, "_preview_impute") with engine.begin() as conn: _clone_table_pair(conn, table, preview_delete, errors_src) @@ -527,17 +530,18 @@ def extract_preview_action(name: str) -> str: return name[idx + len(marker):] return "" -def create_previews_2d(table, row_ids, cols, preview_name_fn, update_errors_fn, update_data_profile_table_fn): +def create_previews_2d(table, row_ids, cols, safe_pg_name_fn, update_errors_fn, update_data_profile_table_fn): """ Create delete, impute_x, and impute_y preview tables for a 2D (two-column) selection. Returns a dict with preview table names and dims=2. """ from app import engine - errors_src = f"errors_{table}" - preview_delete = preview_name_fn(table, "_preview_delete") - preview_impute_x = preview_name_fn(table, "_preview_impute_x") - preview_impute_y = preview_name_fn(table, "_preview_impute_y") + errors_src = f"errors_{table}" + dp_src = f"dp_{table}" + preview_delete_table_name = safe_pg_name_fn(table, "_preview_delete") + preview_impute_x_table_name = safe_pg_name_fn(table, "_preview_impute_x") + preview_impute_y_table_name = safe_pg_name_fn(table, "_preview_impute_y") with engine.begin() as conn: _clone_table_pair(conn, table, preview_delete, errors_src) From f4ea61adf798d0fed77b925496a15b34be5afac8 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 12:32:09 -0600 Subject: [PATCH 22/81] Refactored variable name preview_delete and preview_impute to preview_delete_table_name and preview_impute_table_name --- app/server_utils/service_helpers.py | 53 +++++++++++++++-------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index fb03807..dcd1db3 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -500,25 +500,28 @@ def create_previews_1d(table, row_ids, cols, safe_pg_name_fn, update_errors_fn, preview_delete_table_name = safe_pg_name_fn(table, "_preview_delete") preview_impute_table_name = safe_pg_name_fn(table, "_preview_impute") + # Preview tables are created (error_..._preview, dp_..._preview) with engine.begin() as conn: - _clone_table_pair(conn, table, preview_delete, errors_src) - _clone_table_pair(conn, table, preview_impute, errors_src) + _clone_table_pair(conn, table, preview_delete_table_name, errors_src, dp_src) + _clone_table_pair(conn, table, preview_impute_table_name, errors_src, dp_src) + #create_minimal_preview_table(conn, table, preview_delete, errors_src, cols) #create_minimal_preview_table(conn, table, preview_impute, errors_src, cols) - query.remove_rows_by_ids(table=preview_delete, ids=row_ids) - query.impute_by_ids(table=preview_impute, col=cols[0], ids=row_ids) + # Modify the preview table based on the preview type + query.remove_rows_by_ids(table=preview_delete_table_name, ids=row_ids) + query.impute_by_ids(table=preview_impute_table_name, col=cols[0], ids=row_ids) - errors_df_delete = update_errors_fn(preview_delete) - errors_df_impute = update_errors_fn(preview_impute) - update_data_profile_table_fn(preview_delete, errors_df_delete) - update_data_profile_table_fn(preview_impute, errors_df_impute) + errors_df_delete = update_errors_fn(preview_delete_table_name, cols) + errors_df_impute = update_errors_fn(preview_impute_table_name, cols) + update_data_profile_table_fn(preview_delete_table_name, errors_df_delete, cols) + update_data_profile_table_fn(preview_impute_table_name, errors_df_impute, cols) return { "success": True, - "preview_delete": preview_delete, - "preview_impute": preview_impute, + "preview_delete": preview_delete_table_name, + "preview_impute": preview_impute_table_name, "dims": 1, } @@ -544,27 +547,27 @@ def create_previews_2d(table, row_ids, cols, safe_pg_name_fn, update_errors_fn, preview_impute_y_table_name = safe_pg_name_fn(table, "_preview_impute_y") with engine.begin() as conn: - _clone_table_pair(conn, table, preview_delete, errors_src) - _clone_table_pair(conn, table, preview_impute_x, errors_src) - _clone_table_pair(conn, table, preview_impute_y, errors_src) + _clone_table_pair(conn, table, preview_delete_table_name, errors_src, dp_src) + _clone_table_pair(conn, table, preview_impute_x_table_name, errors_src, dp_src) + _clone_table_pair(conn, table, preview_impute_y_table_name, errors_src, dp_src) - query.remove_rows_by_ids(table=preview_delete, ids=row_ids) - query.impute_by_ids(table=preview_impute_x, col=cols[0], ids=row_ids) - query.impute_by_ids(table=preview_impute_y, col=cols[1], ids=row_ids) + query.remove_rows_by_ids(table=preview_delete_table_name, ids=row_ids) + query.impute_by_ids(table=preview_impute_x_table_name, col=cols[0], ids=row_ids) + query.impute_by_ids(table=preview_impute_y_table_name, col=cols[1], ids=row_ids) - errors_df_delete = update_errors_fn(preview_delete) - errors_df_impute_x = update_errors_fn(preview_impute_x) - errors_df_impute_y = update_errors_fn(preview_impute_y) - update_data_profile_table_fn(preview_delete, errors_df_delete) - update_data_profile_table_fn(preview_impute_x, errors_df_impute_x) - update_data_profile_table_fn(preview_impute_y, errors_df_impute_y) + errors_df_delete = update_errors_fn(preview_delete_table_name, cols) + errors_df_impute_x = update_errors_fn(preview_impute_x_table_name, cols) + errors_df_impute_y = update_errors_fn(preview_impute_y_table_name, cols) + update_data_profile_table_fn(preview_delete_table_name, errors_df_delete, cols) + update_data_profile_table_fn(preview_impute_x_table_name, errors_df_impute_x, cols) + update_data_profile_table_fn(preview_impute_y_table_name, errors_df_impute_y, cols) return { "success": True, - "preview_delete": preview_delete, - "preview_impute_x": preview_impute_x, - "preview_impute_y": preview_impute_y, + "preview_delete": preview_delete_table_name, + "preview_impute_x": preview_impute_x_table_name, + "preview_impute_y": preview_impute_y_table_name, "dims": 2, } From db4ed96298c1b98307503c136f75900114498f1b Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 12:36:15 -0600 Subject: [PATCH 23/81] Added params argument to fetch_sql --- app/db_utils/execute_sql.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/db_utils/execute_sql.py b/app/db_utils/execute_sql.py index 0f5d592..f80cf96 100644 --- a/app/db_utils/execute_sql.py +++ b/app/db_utils/execute_sql.py @@ -12,16 +12,18 @@ def execute_sql(query: str, engine): with engine.begin() as conn: conn.execute(text(query)) -def fetch_sql(query: str, scalar: bool, engine): +def fetch_sql(query: str, scalar: bool, engine, params=None): """ Sends a SQL query to the postgres database. :arg: query: SQL query to execute. :scalar: whether the result from the query will just be 1 row, 1 col, so return as scalar. :return: The result from the query. """ + if params is None: + params = {} with engine.connect() as conn: - result = conn.execute(text(query)) + result = conn.execute(text(query), params) if scalar: return result.scalar() From 5312c77722fa3cd6fc4d4079c1573735c92d4e36 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 13:17:16 -0600 Subject: [PATCH 24/81] Optimized updating of errors table and data profile table. WIP: still need to implement "dirty flags", need to change functions so they have similar structure and need to implement tests --- app/routes/wrangler_routes_sql.py | 80 +++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index 914d919..d798b90 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -17,12 +17,46 @@ Wrangling Endpoints - In-place modification of tables """ +# Where updated_df is just the data that needed to actually be updated +# Assumes that updated_df has the same columns as the target table +# TODO: reimplement with "dirty flags" +def update_table(updated_df, target_table_name, key_col, cols_to_remove): + with engine.begin() as conn: + result = conn.execute( + text(f'DELETE FROM "{target_table_name}" WHERE "{key_col}" = ANY(:categories)'), + {"categories": cols_to_remove} + ) + + staging_table = _safe_pg_name(target_table_name, "_staging") + + cols_to_update = updated_df.columns + + # 1. Push data to a temp staging table + updated_df.to_sql(staging_table, engine, if_exists='replace', index=False) + + # Make sure that there's a main errors table we can update + inspector = inspect(engine) + assert inspector.has_table(target_table_name), f"Table {target_table_name} does not exist!" + + # 2. Set-based update, Postgres native syntax + with engine.begin() as conn: + set_clause = ", ".join(f'"{c}" = staged."{c}"' for c in cols_to_update) + conn.execute(text(f''' + UPDATE "{target_table_name}" target + SET {set_clause} + FROM "{staging_table}" staged + WHERE target."{key_col}" = staged."{key_col}" + ''')) + + conn.execute(text(f'DROP TABLE "{staging_table}"')) + + # ───────────────────────────────────────────────────────────────────────────── # Helper: Re-run error detection after modification # ───────────────────────────────────────────────────────────────────────────── # Returns error_df for update_data_profile_table to use (so it doesn't have to get it from the database) -def update_errors_table(table_name: str) -> pd.DataFrame: +def update_errors_table(table_name: str, columns_selected_for_wrangling: list) -> pd.DataFrame: # TODO: fix this so it doesn't update the whole table after small changes to the table """ After modifying a table in-place, re-run error detection @@ -30,13 +64,24 @@ def update_errors_table(table_name: str) -> pd.DataFrame: """ try: df = pd.read_sql_query(f'SELECT * FROM "{table_name}"', engine) + + # TODO: optimize this so it doesn't load the whole table into a df first + df = df[columns_selected_for_wrangling] + detected_errors_df = run_detectors(df) errors_table_name = f"errors_{table_name}" + + key_column = "column_id" + update_table(detected_errors_df, errors_table_name, key_column, columns_selected_for_wrangling) + # Drop first via raw SQL to avoid SQLAlchemy reflection (which fails on # table names > 63 chars due to PostgreSQL identifier truncation). - with engine.begin() as conn: - conn.execute(sa_text(f'DROP TABLE IF EXISTS "{errors_table_name}"')) - detected_errors_df.to_sql(errors_table_name, engine, if_exists='fail', index=False) + #with engine.begin() as conn: + # conn.execute(sa_text(f'DROP TABLE IF EXISTS "{errors_table_name}"')) + + + # detected_errors_df.to_sql(errors_table_name, engine, if_exists='fail', index=False) + print(f"✓ Updated errors table: {errors_table_name}") return detected_errors_df except Exception as e: @@ -44,17 +89,24 @@ def update_errors_table(table_name: str) -> pd.DataFrame: traceback.print_exc() raise -def update_data_profile_table(table_name: str, error_df: pd.DataFrame) -> None: +# TODO: Make update_data_profile_table and update_errors_table more similar +# TODO: Re-implement with "dirty flags" +def update_data_profile_table(table_name: str, error_df: pd.DataFrame, columns_selected_for_wrangling: list) -> None: try: - data_profile_df = create_data_profile_df(table_name, engine, error_df) - print("CALCULATED DATA PROFILE DF SUCCESSFULLY:") - print(data_profile_df) - data_profile_table_name = f"dp_{table_name}" - with engine.begin() as conn: - conn.execute(sa_text(f"DROP TABLE IF EXISTS {data_profile_table_name}")) - data_profile_df.to_sql(data_profile_table_name, engine, if_exists='fail', index=False) - - print(f"✓ Updated data profile table: {data_profile_table_name}") + + # TODO: optimize this so it doesn't load the whole table into a df first + dp_table_name = f"dp_{table_name}" + print("COL NAMES", columns_selected_for_wrangling) + + updated_df = create_data_profile_df(table_name, engine, col_names=columns_selected_for_wrangling, error_df=error_df) + + key_column = "column_name" + + update_table(updated_df, dp_table_name, key_column, columns_selected_for_wrangling) + + + + print(f"✓ Updated data profile table: {dp_table_name}") except Exception as e: print(f"ERROR: Could not update data profile table for {table_name}: {e}") traceback.print_exc() From b95e8f383fd525998bae0e498782299f0fa8251f Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 14:07:42 -0600 Subject: [PATCH 25/81] Import text from sqlalchemy --- app/routes/wrangler_routes_sql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index d798b90..3b8fc20 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -10,6 +10,7 @@ from app.server_utils.service_helpers import run_detectors, create_previews_1d, create_previews_2d, \ execute_wrangle_preview, _safe_pg_name, create_data_profile_df from sqlalchemy import text as sa_text +from sqlalchemy import inspect, text From 93cc0ba7d18d260e9c45661d32e7d6a33f1f4ec8 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 14:23:04 -0600 Subject: [PATCH 26/81] Added documentation to data_profile.py --- app/db_utils/data_profile.py | 115 ++++++++++++++++++++++++++++++++--- 1 file changed, 105 insertions(+), 10 deletions(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index f2b8b2a..ef26d1c 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -26,7 +26,20 @@ def to_scalar(val): return val class DataProfile: + """ + Class that handles queries to get summary stats about the main data table. + Should be able to work without using main_df and error_df at all. + This class prioritizes SQL queries and when those don't work, the table is loaded and the pd dataframe method is used + instead. + """ def __init__(self, table_name, engine=None, main_df=None, error_df=None): + """ + :param table_name: name of the main data table + :param engine: + :param main_df:the main data table as a data frame + :param error_df:the error data table as a data frame + """ + self.table_name = table_name self.data_profile_table_name = "dp" + table_name self.engine = engine @@ -53,31 +66,49 @@ def __init__(self, table_name, engine=None, main_df=None, error_df=None): } def get_error_df(self): + """ + Gets the error df by loading it first (making sure it's not None) then returning it + :return: The error_df + """ self.load_error_df() return self._error_df def get_main_df(self): + """ + Gets the main df by loading it first (making sure it's not None) then returning it + :return: The main_df + """ self.load_main_df() return self._main_df def load_error_df(self): + """ + Loads the error_df from the table if it wasn't passed in as an argument + :return: None + """ if self._error_df is None: assert self.engine is not None, f"engine cannot be None if error_df is None" #self.error_df = load_table_to_df(f"errors_{self.table_name}", self.engine) self._error_df = pd.read_sql_query(f'SELECT * FROM "{"errors_" + self.table_name}"', self.engine) def load_main_df(self): + """ + Loads the main_df from the table if it wasn't passed in as an argument + :return: None + """ if self._main_df is None: assert self.engine is not None, f"engine cannot be None if main_df is None" - #self.main_df = load_table_to_df(self.table_name, self.engine) self._main_df = pd.read_sql_query(f'SELECT * FROM "{self.table_name}"', self.engine) - - - def get_col_data(self, column_name, attribute_name): + def get_processed_column_data(self, column_name, attribute_name): + """ + :param column_name: Name of the column to get data for + :param attribute_name: Name of the attribute (used to determine how to process the data) + :return: Processed column data + """ assert (attribute_name in self.attribute_type_assignment['categorical'] or attribute_name in self.attribute_type_assignment['numeric']), f"Invalid attribute name {attribute_name}" if attribute_name in self.attribute_type_assignment['categorical']: @@ -89,6 +120,10 @@ def get_col_data(self, column_name, attribute_name): # TODO: Make the sql query for this work def get_col_names(self): + """ + self: DataProfile instance + :return: List of column names in the main data table + """ try: query = 'SELECT column_name FROM information_schema.columns WHERE table_name = :table_name ORDER BY ordinal_position' params = {"table_name": self.table_name} @@ -97,6 +132,7 @@ def get_col_names(self): col_names = [] for row in result: + # For some reason the SQL query returns some unwanted columns so I'm taking them out if row[0] not in ['index', 'level_0', ]: col_names.append(row[0]) @@ -114,6 +150,12 @@ def get_col_names(self): return col_names def look_up_stat_from_profile(self, attribute_name, column_name): + """ + Look up a summary statistic for a specific column from the data profile table. + :param attribute_name: Name of the attribute (e.g., 'mean', 'median', 'min', 'max', etc.) + :param column_name: Name of the column for which the statistic is being looked up. + :return: The value of the statistic if found, otherwise None. + """ data_profile_table_name = "dp_" + self.table_name try: query = f'SELECT "{attribute_name}" FROM "{data_profile_table_name}" WHERE "column_name" = :column_name' @@ -123,13 +165,26 @@ def look_up_stat_from_profile(self, attribute_name, column_name): return stat except Exception as e: print(f"Error querying attribute from data profile table: {e}") + return None + def calculate_summary_stat_using_sql(self, stat_query, column_name): + """ + :param stat_query: The SQL aggregate function to use (e.g., 'AVG', 'MIN', 'MAX', etc.) + :param column_name: Name of the column for which the statistic is being looked up. + :return: The value of the statistic if found, otherwise None. + """ query = f'SELECT {stat_query}("{column_name}") FROM "{self.table_name}"' stat = fetch_sql(query, True, self.engine) return stat def calculate_column_attribute(self, attribute_name, column_name, look_up_stat=True): + """ + :param attribute_name: Name of the attribute (e.g., 'mean', 'median', 'min', 'max') + :param column_name: Name of the column for which the statistic is being looked up. + :param look_up_stat: If True, first attempt to look up the statistic from the data profile table. If False, calculate it directly. + :return: The value of the statistic if found or calculated, otherwise None. + """ if look_up_stat: look_up_value = self.look_up_stat_from_profile(attribute_name, column_name) @@ -142,7 +197,12 @@ def calculate_column_attribute(self, attribute_name, column_name, look_up_stat=T # TODO: save stat off to table if newly calculated def _calculate_mean(self, column_name): - col_data = self.get_col_data(column_name, 'mean') + """ + :param column_name: Name of the column for which the mean is being calculated + :return: The mean + """ + col_data = self.get_processed_column_data(column_name, 'mean') + # Try get the mean from SQL first, if that fails, calculate it manually using the data frame try: avg = self.calculate_summary_stat_using_sql('AVG', column_name) except Exception as e: @@ -158,8 +218,13 @@ def _calculate_mean(self, column_name): def _calculate_median(self, column_name): - col_data = self.get_col_data(column_name, 'median') + """ + :param column_name: Name of the column for which the median is being calculated + :return: The median + """ + col_data = self.get_processed_column_data(column_name, 'median') + # Try get the median from SQL first, if it fails, calculate manually using data frame try: query = f'SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY "{column_name}") FROM "{self.table_name}"' median = fetch_sql(query, True, self.engine) @@ -177,7 +242,13 @@ def _calculate_median(self, column_name): def _calculate_max(self, column_name): - col_data = self.get_col_data(column_name, 'max') + """ + :param column_name: Name of the column for which the max is being calculated + :return: The maximum + """ + + col_data = self.get_processed_column_data(column_name, 'max') + # Try using SQL query, if it fails, calculate manually using data frame try: maximum = self.calculate_summary_stat_using_sql('MAX', column_name) except Exception as e: @@ -193,8 +264,13 @@ def _calculate_max(self, column_name): return maximum def _calculate_min(self, column_name): + """ + :param column_name: Name of the column for which the minimum is being calculated + :return: The minimum + """ + col_data = self.get_processed_column_data(column_name, 'min') - col_data = self.get_col_data(column_name, 'min') + # Try using SQL query, if it fails, calculate manually using data frame try: minimum = self.calculate_summary_stat_using_sql('MIN', column_name) except Exception as e: @@ -210,7 +286,13 @@ def _calculate_min(self, column_name): return minimum def _calculate_num_categories(self, column_name): - col_data = self.get_col_data(column_name, 'n_categories') + """ + :param column_name: Name of the column for which the number of categories is being calculated + :return: The number of categories + """ + col_data = self.get_processed_column_data(column_name, 'n_categories') + + # Try using SQL query, if it fails, calculate manually using data frame try: query = f'SELECT COUNT(DISTINCT "{column_name}") FROM "{self.table_name}"' n_categories = fetch_sql(query, True, self.engine) @@ -230,9 +312,14 @@ def _calculate_num_categories(self, column_name): def _calculate_mode(self, column_name): + """ + :param column_name: Name of the column for which the mode is being calculated + :return: The mode + """ + print("Calculating mode manually using data...") self.load_main_df() - col_data = self.get_col_data(column_name, 'mode') + col_data = self.get_processed_column_data(column_name, 'mode') mode = col_data.mode() return mode @@ -240,6 +327,10 @@ def _calculate_mode(self, column_name): # Functions relating to error info # Dict mapping from error type to total error count def _calculate_error_count_dict(self, column_name): + """ + :param column_name: Name of the column for which the error count is being calculated + :return: The error count dict ({"missing": 10, "mismatch": 5, ...}) + """ try: query = f'SELECT {column_name}, COUNT(*) as cnt FROM {self.table_name} GROUP BY {column_name} ORDER BY cnt DESC' category_counts = dict(fetch_sql(query, True, self.engine)) @@ -262,6 +353,10 @@ def _calculate_error_count_dict(self, column_name): # Dict mapping from class to error types to error counts # TODO: Implement SQL query version def _calculate_class_error_count_dict(self, column_name): + """ + :param column_name: Name of the column for which the class error count is being calculated + :return: The class error count dict ({"Male": {"missing": 10, "mismatch": 5, ...}, "Female": {"missing": 10, "mismatch": 5, ...}}) + """ print("Calculating class error counts manually using data...") self.load_error_df() From a949d201c628df3f688a6c77dbf2cd9f3859a673 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 14:23:30 -0600 Subject: [PATCH 27/81] Added documentation to data_profile.py --- app/db_utils/data_profile.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index ef26d1c..fa7edca 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -6,10 +6,7 @@ from app.db_utils.execute_sql import fetch_sql - - -#TODO: add documentation - +# TODO: is this needed? this may be a duplicate def to_scalar(val): if val is None: return None @@ -316,6 +313,7 @@ def _calculate_mode(self, column_name): :param column_name: Name of the column for which the mode is being calculated :return: The mode """ + # TODO: add a SQL query version to calculate the mode print("Calculating mode manually using data...") self.load_main_df() @@ -357,6 +355,7 @@ def _calculate_class_error_count_dict(self, column_name): :param column_name: Name of the column for which the class error count is being calculated :return: The class error count dict ({"Male": {"missing": 10, "mismatch": 5, ...}, "Female": {"missing": 10, "mismatch": 5, ...}}) """ + # TODO: Implement SQL query version print("Calculating class error counts manually using data...") self.load_error_df() From 5281a2bb940163ba2e845ede243cf3612d03a257 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 5 Jul 2026 14:38:02 -0600 Subject: [PATCH 28/81] Added documentation in service_helpers for create_data_profile_df() --- app/server_utils/service_helpers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index dcd1db3..6503b59 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -179,6 +179,14 @@ def run_detectors(data_frame): # TODO: CLEAN UP THIS FUNCTION # TODO: Test this function!!! (Write test for it) def create_data_profile_df(table_name, engine, col_names=None, error_df=None, main_df=None): + """ + :param table_name: the name of the table in the database + :param engine: the engine to use + :param col_names: the column names of interest in the table + :param error_df: the error dataframe (optional) + :param main_df: the main dataframe (optional) + :return: a dataframe of the data profile for the table + """ print("CREATED DATA_PROFILE DF FOR TABLE", table_name) data_profile = DataProfile(table_name, engine=engine, main_df=main_df, error_df=error_df) From 9f8dee71cd8111f676d18f79e1aee16f71c3b584 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 10 Jul 2026 17:35:04 -0600 Subject: [PATCH 29/81] Moved ColumnTypes class into data_profile.py and added column types functions into the DataProfile class --- app/db_utils/data_profile.py | 155 ++++++++++++++++++++++++++++++----- 1 file changed, 136 insertions(+), 19 deletions(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index fa7edca..5f29853 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -22,6 +22,102 @@ def to_scalar(val): return val.item() return val +""" +--- ColumnTypes --- +Inspects a table's schema to classify each column as numeric, categorical, or mixed-type. +""" + +class ColumnTypes: + def __init__(self, main_table_name: str, engine): + self.numeric_cols = set() + self.categorical_mixed = set() + self.pure_categorical = set() + self.engine = engine + self.gather_numeric_cols(main_table_name) + self.gather_mixed_cols(main_table_name) + + + def gather_numeric_cols(self, main_table_name: str): + """ + Distinguishes the numeric columns from the categorical columns. + :arg: main_table_name: name of the main table. + """ + + + fetch_col_types = f'''SELECT column_name, data_type + FROM information_schema.columns + WHERE table_name = '{main_table_name}';''' + + fetched_rows = fetch_sql(fetch_col_types, False, self.engine) + if fetched_rows: + numeric_types = { + 'integer', 'bigint', 'numeric', + 'real', 'double precision', 'smallint' + } + + for row in fetched_rows: + col_name = row[0] + data_type = row[1] + + if data_type in numeric_types: + self.numeric_cols.add(col_name) + else: + self.categorical_mixed.add(col_name) + else: + raise Exception(f"No rows fetched from table: {main_table_name}") + + + def gather_mixed_cols(self, main_table_name: str): + """ + Gather the columns that are labeled as categorical but contain numeric data as well. + :arg: main_table_name: name of the main table. + """ + + # There are no categorical columns in the dataset. + if len(self.categorical_mixed) == 0: + return + + numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" + + # Initialized in the other constructor func gather_numeric_cols. This starts as all categorical columns. + # Stop early if a mixed type is found, since that makes the entire column of mixed type. + queries = [ + f"""( + SELECT '{col}' AS column_name + FROM "{main_table_name}" + WHERE "{col}" ~ {numeric_regex} + LIMIT 1 + )""" + for col in self.categorical_mixed + ] + + fetch_mixed_types = "\nUNION ALL\n".join(queries) + mixed_cols = fetch_sql(fetch_mixed_types, False, self.engine) + + mixed_col_names = set(row[0] for row in mixed_cols) if mixed_cols else set() + self.pure_categorical = self.categorical_mixed - mixed_col_names + self.categorical_mixed = mixed_col_names + + def is_categorical_col(self, col_name: str): + return col_name in self.pure_categorical + + def is_numeric_col(self, col_name: str): + """ + Determines whether the given column from the table used to construct this class is numeric. + :arg: col_name: name of the column (assumes it is from the same table used to construct this class). + :return: whether the given col_name is numeric. + """ + return col_name in self.numeric_cols + + + def is_mixed_col(self, col_name: str): + """ + Determines whether the given column from the table used to construct this class is of mixed type. + :arg: col_name: name of the column (assumes it is from the same table used to construct this class). + :return: whether the given col_name is of mixed type. + """ + return col_name in self.categorical_mixed + class DataProfile: """ Class that handles queries to get summary stats about the main data table. @@ -348,29 +444,50 @@ def _calculate_error_count_dict(self, column_name): return category_counts - # Dict mapping from class to error types to error counts - # TODO: Implement SQL query version - def _calculate_class_error_count_dict(self, column_name): + def get_col_type(self, column_name): """ - :param column_name: Name of the column for which the class error count is being calculated - :return: The class error count dict ({"Male": {"missing": 10, "mismatch": 5, ...}, "Female": {"missing": 10, "mismatch": 5, ...}}) + :param column_name: Name of the column for which the type is being checked + :return: The type of the column in a string """ - # TODO: Implement SQL query version - print("Calculating class error counts manually using data...") + if self.col_types.is_numeric_col(column_name): + return "numeric" + elif self.col_types.is_categorical_col(column_name): + return "categorical" + elif self.col_types.is_mixed_col(column_name): + return "mixed" + else: + return None - self.load_error_df() + def is_numeric_col(self, column_name): + """ + :param column_name: Name of the column for which the type is being checked + :return: True if the column is numeric, False otherwise + """ + return self.col_types.is_numeric_col(column_name) + + def is_categorical_col(self, column_name): + """ + :param column_name: Name of the column for which the type is being checked + :return: True if the column is categorical, False otherwise + """ + return self.col_types.is_categorical_col(column_name) + + def is_mixed_col(self, column_name): + """ + :param column_name: Name of the column for which the type is being checked + :return: True if the column is mixed, False otherwise + """ + return self.col_types.is_mixed_col(column_name) + + + def get_column_names(self): + """ + :return: List of column names + """ + # uses the sets of column names from the ColumnTypes class to get all column names + all_cols = [list(self.col_types.numeric_cols), list(self.col_types.pure_categorical), + list(self.col_types.categorical_mixed)] - counts_by_column = {} - if not self._error_df.empty: # If error_df is empty (no errors in data selection) - counts_by_column = ( - self._error_df.groupby(['column_id', 'error_type']) - .size() - .unstack(fill_value=0) - .to_dict(orient='index') - ) - if counts_by_column is not None: - counts_by_column = json.dumps(counts_by_column) - return counts_by_column From 44c1cf7098d6694d120e5e8d1d6246f40b8fcc95 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 10 Jul 2026 17:38:00 -0600 Subject: [PATCH 30/81] Removed ColumnTypes from db_functions_sql.py and added data profile instance into DBOperations to get column type functions --- app/db_utils/db_functions_sql.py | 138 ++++++------------------------- 1 file changed, 24 insertions(+), 114 deletions(-) diff --git a/app/db_utils/db_functions_sql.py b/app/db_utils/db_functions_sql.py index 86cc55f..4846d8b 100644 --- a/app/db_utils/db_functions_sql.py +++ b/app/db_utils/db_functions_sql.py @@ -5,111 +5,18 @@ from app.server_utils import service_helpers from app.db_utils.filtering_sql import FilteringSQL from app.db_utils.execute_sql import fetch_sql, execute_sql +from app.db_utils.data_profile import DataProfile """ Provides two classes for querying and visualizing data from a PostgreSQL database table, with support for data filtering and error annotation overlays on all chart types. ---- ColumnTypes --- -Inspects a table's schema to classify each column as numeric, categorical, or mixed-type. - --- DBOperations --- Wraps all core DB operations for a single primary table. Builds and executes multi-step CTE SQL queries that produce JSON payloads for 1D histograms, 2D histograms, and scatterplots, each annotated with per-bin/per-point error breakdowns. Also manages row-level data filters. """ -class ColumnTypes: - def __init__(self, main_table_name: str, engine): - self.numeric_cols = set() - self.categorical_mixed = set() - self.pure_categorical = set() - self.engine = engine - self.gather_numeric_cols(main_table_name) - self.gather_mixed_cols(main_table_name) - - - def gather_numeric_cols(self, main_table_name: str): - """ - Distinguishes the numeric columns from the categorical columns. - :arg: main_table_name: name of the main table. - """ - - - fetch_col_types = f'''SELECT column_name, data_type - FROM information_schema.columns - WHERE table_name = '{main_table_name}';''' - - fetched_rows = fetch_sql(fetch_col_types, False, self.engine) - if fetched_rows: - numeric_types = { - 'integer', 'bigint', 'numeric', - 'real', 'double precision', 'smallint' - } - - for row in fetched_rows: - col_name = row[0] - data_type = row[1] - - if data_type in numeric_types: - self.numeric_cols.add(col_name) - else: - self.categorical_mixed.add(col_name) - else: - raise Exception(f"No rows fetched from table: {main_table_name}") - - - def gather_mixed_cols(self, main_table_name: str): - """ - Gather the columns that are labeled as categorical but contain numeric data as well. - :arg: main_table_name: name of the main table. - """ - - # There are no categorical columns in the dataset. - if len(self.categorical_mixed) == 0: - return - - numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" - - # Initialized in the other constructor func gather_numeric_cols. This starts as all categorical columns. - # Stop early if a mixed type is found, since that makes the entire column of mixed type. - queries = [ - f"""( - SELECT '{col}' AS column_name - FROM "{main_table_name}" - WHERE "{col}" ~ {numeric_regex} - LIMIT 1 - )""" - for col in self.categorical_mixed - ] - - fetch_mixed_types = "\nUNION ALL\n".join(queries) - mixed_cols = fetch_sql(fetch_mixed_types, False, self.engine) - - mixed_col_names = set(row[0] for row in mixed_cols) if mixed_cols else set() - self.pure_categorical = self.categorical_mixed - mixed_col_names - self.categorical_mixed = mixed_col_names - - def is_categorical_col(self, col_name: str): - return col_name in self.pure_categorical - - def is_numeric_col(self, col_name: str): - """ - Determines whether the given column from the table used to construct this class is numeric. - :arg: col_name: name of the column (assumes it is from the same table used to construct this class). - :return: whether the given col_name is numeric. - """ - return col_name in self.numeric_cols - - - def is_mixed_col(self, col_name: str): - """ - Determines whether the given column from the table used to construct this class is of mixed type. - :arg: col_name: name of the column (assumes it is from the same table used to construct this class). - :return: whether the given col_name is of mixed type. - """ - return col_name in self.categorical_mixed - # Wraps up all Core DBOperations into one class using a primary main_table. class DBOperations: @@ -123,7 +30,8 @@ def __init__(self, engine): self.engine = engine self.main_table_name = None self.error_table_name = None - self.col_types = None + self.data_profile_table_name = None + self.data_profile = None self.filtering_table = None self.active_hists = {} @@ -134,11 +42,12 @@ def reset(self): """ self.main_table_name = None self.error_table_name = None - self.col_types = None + self.data_profile_table_name = None + self.data_profile = None self.filtering_table = None self.active_hists = {} - def load_table(self, main_table_name: str, error_table_name: str = None): + def load_table(self, main_table_name: str, error_table_name: str = None, data_profile_table_name: str = None): """ Loads in the main and error tables, inits the ColumnTypes and FilteringSQL objects with the new table @@ -147,7 +56,8 @@ def load_table(self, main_table_name: str, error_table_name: str = None): """ self.main_table_name = main_table_name self.error_table_name = error_table_name if error_table_name is not None else "errors_" + main_table_name - self.col_types = ColumnTypes(main_table_name, self.engine) + self.data_profile_table_name = data_profile_table_name if data_profile_table_name is not None else "dp_" + main_table_name + self.data_profile = DataProfile(main_table_name, self.engine) self.filtering_table = FilteringSQL(main_table_name, self.engine) self.active_hists = {} @@ -299,7 +209,7 @@ def gather_bins_1d_hist(self, axis_column: str, bin_count: int) -> str: :return: the query for the binning. """ - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" bin_logic = f'''CASE WHEN d.value::text ~ {numeric_regex} THEN @@ -387,7 +297,7 @@ def build_numeric_scale_data_1d_hist(self, axis_column: str, bin_count: int) -> :return: the query for the numeric scaling data. """ - if not self.col_types.is_numeric_col(axis_column): + if not self.data_profile.is_numeric_col(axis_column): return "" else: return f''', range_data AS ( @@ -422,7 +332,7 @@ def construct_1d_hist_json(self, axis_column: str) -> str: binned_data = '''SELECT (SELECT json_agg(json_build_array("ID", bin)) FROM binned_data),''' - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): return f'''{binned_data} (SELECT json_build_object( 'histograms', -- For numeric: handle mixed bins (numeric and "null") - keep bins as text @@ -520,7 +430,7 @@ def generate_2d_hist_bounds(self, bound_table_name: str, axis_column: str, col_a :return: the query to generate the bound tables. """ - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" return f''', {bound_table_name} AS ( SELECT @@ -554,7 +464,7 @@ def gather_bins_2d_hist(self, x_axis_column: str, y_axis_column: str, x_bin_coun for axis_column, bin_count, axis_alias, bounding_table in axis_info: numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): bin_logic = f'''CASE WHEN d.{axis_alias}::text ~ {numeric_regex} THEN -- Clamp bin number to 0..(bin_count-1) range @@ -654,7 +564,7 @@ def build_numeric_scale_data_2d_hist(self, bound_table_name: str, axis_column: s :return: the query for the numeric scaling data. """ - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): return f''', {scale_table_name}_range_data AS ( SELECT min_val, @@ -687,7 +597,7 @@ def construct_2d_hist_json(self, x_axis_column: str, y_axis_column: str, x_scale empty_set = r"'{}'" # Handles mixed types in x-axis. - if self.col_types.is_numeric_col(x_axis_column): + if self.data_profile.is_numeric_col(x_axis_column): json_x_type = f'''CASE WHEN x_bin ~ {numeric_regex} THEN 'numeric' ELSE 'categorical' END''' json_order_by_x = f'''CASE WHEN x_bin ~ {numeric_regex} THEN lpad(x_bin, 10, '0') ELSE x_bin END''' else: @@ -695,7 +605,7 @@ def construct_2d_hist_json(self, x_axis_column: str, y_axis_column: str, x_scale json_order_by_x = "x_bin" # Handles mixed types in y-axis. - if self.col_types.is_numeric_col(y_axis_column): + if self.data_profile.is_numeric_col(y_axis_column): json_y_type = f'''CASE WHEN y_bin ~ {numeric_regex} THEN 'numeric' ELSE 'categorical' END''' json_order_by_y = f'''CASE WHEN y_bin ~ {numeric_regex} THEN lpad(y_bin, 10, '0') ELSE y_bin END''' else: @@ -724,7 +634,7 @@ def construct_2d_hist_json(self, x_axis_column: str, y_axis_column: str, x_scale for i in range(len(json_scale_data)): scale_label, axis_column, scale_table_name, axis_bin = json_scale_data[i] - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): axis_numeric_info = f'''(SELECT COALESCE(json_agg(json_build_object('x0', x0, 'x1', x1) ORDER BY bin_num), '[]'::json) FROM {scale_table_name})''' else: @@ -871,7 +781,7 @@ def collect_scatter_axis_bounds(self, bound_table_name: str, axis_column: str, c :return: the query for aggregating scatterplot error data w/ sampled points. """ - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): return f''', {bound_table_name} AS ( SELECT @@ -900,9 +810,9 @@ def construct_scatter_json(self, x_axis_column: str, y_axis_column: str, x_col_a # Helper function to determine axis type def determine_axis_type(axis_column: str, col_alias: str) -> str: - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): return "ELSE 'numeric'" - elif self.col_types.is_mixed_col(axis_column): + elif self.data_profile.is_mixed_col(axis_column): return f"WHEN ({col_alias}::text ~ {numeric_regex}) THEN 'numeric' ELSE 'categorical'" else: return "ELSE 'categorical'" @@ -910,9 +820,9 @@ def determine_axis_type(axis_column: str, col_alias: str) -> str: # Helper function to determine JSON axis type def determine_json_axis_type(axis_column: str, col_alias: str) -> str: - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): return f"ELSE to_json({col_alias}::numeric)" - elif self.col_types.is_mixed_col(axis_column): + elif self.data_profile.is_mixed_col(axis_column): return f"WHEN ({col_alias}::text ~ {numeric_regex}) THEN to_json({col_alias}::numeric) ELSE to_json({col_alias}::text)" else: return f"ELSE to_json({col_alias}::text)" @@ -958,12 +868,12 @@ def determine_json_axis_type(axis_column: str, col_alias: str) -> str: for i in range(len(json_scale_data)): scale_label, axis_column, bounding_table, axis_alias = json_scale_data[i] - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): axis_numeric_info = f'''json_build_array( (SELECT min_val FROM {bounding_table}), (SELECT max_val + 1 FROM {bounding_table}) )''' - elif self.col_types.is_mixed_col(axis_column): + elif self.data_profile.is_mixed_col(axis_column): axis_numeric_info = f'''json_build_array( (SELECT COALESCE(MIN({axis_alias}::numeric), 0) FROM sampled_data WHERE {axis_alias}::text ~ {numeric_regex}), From aa2af9ca1c9ff60942223a4c43055366b668e1ea Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 10 Jul 2026 17:51:55 -0600 Subject: [PATCH 31/81] Removed pandas dataframe alternate in data profile calculations (included removing passing in of main_df and error_df). Removed class error counts function and replaced it with category count function --- app/db_utils/data_profile.py | 206 +++++++++--------- app/routes/wrangler_routes_sql.py | 5 + .../data_attribute_summary_integration.py | 21 +- app/server_utils/service_helpers.py | 1 - 4 files changed, 120 insertions(+), 113 deletions(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index 5f29853..b5cd646 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -132,14 +132,15 @@ def __init__(self, table_name, engine=None, main_df=None, error_df=None): :param main_df:the main data table as a data frame :param error_df:the error data table as a data frame """ - self.table_name = table_name - self.data_profile_table_name = "dp" + table_name + self.data_profile_table_name = "dp_" + table_name + self.error_table_name = "errors_" + table_name + self.col_types = ColumnTypes(table_name, engine) + + self.engine = engine # IMPORTANT: main_df and error_df are NOT guaranteed to not be None (so that they don't have to be loaded each time for efficiency) # Use get_main_df and get_error_df instead of accessing them directly - self._main_df = main_df - self._error_df = error_df self.default_attributes = ['mean', 'median', 'min', 'max', 'n_categories', 'mode', 'error_counts', 'class_error_counts'] self.name_to_func = { 'mean': self._calculate_mean, @@ -158,59 +159,12 @@ def __init__(self, table_name, engine=None, main_df=None, error_df=None): 'mode', 'error_counts', 'class_error_counts'], } - def get_error_df(self): - """ - Gets the error df by loading it first (making sure it's not None) then returning it - :return: The error_df - """ - self.load_error_df() - return self._error_df + self.dtype_dict = None - def get_main_df(self): - """ - Gets the main df by loading it first (making sure it's not None) then returning it - :return: The main_df - """ - self.load_main_df() - return self._main_df - def load_error_df(self): - """ - Loads the error_df from the table if it wasn't passed in as an argument - :return: None - """ - if self._error_df is None: - assert self.engine is not None, f"engine cannot be None if error_df is None" - #self.error_df = load_table_to_df(f"errors_{self.table_name}", self.engine) - self._error_df = pd.read_sql_query(f'SELECT * FROM "{"errors_" + self.table_name}"', self.engine) - - def load_main_df(self): - """ - Loads the main_df from the table if it wasn't passed in as an argument - :return: None - """ - if self._main_df is None: - assert self.engine is not None, f"engine cannot be None if main_df is None" - self._main_df = pd.read_sql_query(f'SELECT * FROM "{self.table_name}"', self.engine) - - def get_processed_column_data(self, column_name, attribute_name): - """ - :param column_name: Name of the column to get data for - :param attribute_name: Name of the attribute (used to determine how to process the data) - :return: Processed column data - """ - assert (attribute_name in self.attribute_type_assignment['categorical'] or attribute_name in self.attribute_type_assignment['numeric']), f"Invalid attribute name {attribute_name}" - - if attribute_name in self.attribute_type_assignment['categorical']: - col_data = self._main_df[column_name].fillna('N/A') - if attribute_name in self.attribute_type_assignment['numeric']: - col_data = pd.to_numeric(self._main_df[column_name], errors='coerce').dropna() - - return col_data - # TODO: Make the sql query for this work def get_col_names(self): """ @@ -234,11 +188,6 @@ def get_col_names(self): except Exception as e: print(f"AHHHHHHHHHH Querying for col names unsuccessful because of error: {e}") - print("Getting col names from Data frame instead") - self.load_main_df() - print("COL NAMES FROM main_df.columns:", self._main_df.columns) - - col_names = self._main_df.columns return col_names @@ -294,18 +243,14 @@ def _calculate_mean(self, column_name): :param column_name: Name of the column for which the mean is being calculated :return: The mean """ - col_data = self.get_processed_column_data(column_name, 'mean') + # Try get the mean from SQL first, if that fails, calculate it manually using the data frame try: avg = self.calculate_summary_stat_using_sql('AVG', column_name) except Exception as e: print(f"Error fetching the mean for table {self.table_name} at column {column_name}: {e}") + avg = None - print("Calculating mean manually using data...") - self.load_main_df() - - avg = col_data.mean() - print(f"Updating mean value for column {column_name} in data profile") return avg @@ -315,21 +260,14 @@ def _calculate_median(self, column_name): :param column_name: Name of the column for which the median is being calculated :return: The median """ - col_data = self.get_processed_column_data(column_name, 'median') # Try get the median from SQL first, if it fails, calculate manually using data frame try: query = f'SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY "{column_name}") FROM "{self.table_name}"' median = fetch_sql(query, True, self.engine) except Exception as e: - self.load_main_df() print(f"Error fetching the median for table {self.table_name} at column {column_name}: {e}") - - print("Calculating median manually using data...") - - median = col_data.median() - - print(f"Updating median value for column {column_name} in data profile") + median = None return median @@ -340,19 +278,13 @@ def _calculate_max(self, column_name): :return: The maximum """ - col_data = self.get_processed_column_data(column_name, 'max') # Try using SQL query, if it fails, calculate manually using data frame try: maximum = self.calculate_summary_stat_using_sql('MAX', column_name) except Exception as e: print(f"Error fetching the maximum for table {self.table_name} at column {column_name}: {e}") - print("Calculating maximum manually using data...") - self.load_main_df() - - maximum = col_data.max() - - print(f"Updating maximum value for column {column_name} in data profile") + maximum = None return maximum @@ -361,7 +293,6 @@ def _calculate_min(self, column_name): :param column_name: Name of the column for which the minimum is being calculated :return: The minimum """ - col_data = self.get_processed_column_data(column_name, 'min') # Try using SQL query, if it fails, calculate manually using data frame try: @@ -369,12 +300,7 @@ def _calculate_min(self, column_name): except Exception as e: print(f"Error fetching the minimum for table {self.table_name} at column {column_name}: {e}") - print("Calculating minimum manually using data...") - self.load_main_df() - - minimum = col_data.min() - - print(f"Updating minimum value for column {column_name} in data profile") + minimum = None return minimum @@ -383,7 +309,6 @@ def _calculate_num_categories(self, column_name): :param column_name: Name of the column for which the number of categories is being calculated :return: The number of categories """ - col_data = self.get_processed_column_data(column_name, 'n_categories') # Try using SQL query, if it fails, calculate manually using data frame try: @@ -394,12 +319,7 @@ def _calculate_num_categories(self, column_name): print("AHHH SQL QUERY DIDN'T WORK") print(f"Error fetching the n_categories for table {self.table_name} at column {column_name}: {e}") - print("Calculating n_categories manually using data...") - self.load_main_df() - - n_categories = col_data.nunique() - - print(f"Updating n_categories value for column {column_name} in data profile") + n_categories = None return n_categories @@ -409,12 +329,23 @@ def _calculate_mode(self, column_name): :param column_name: Name of the column for which the mode is being calculated :return: The mode """ - # TODO: add a SQL query version to calculate the mode + try: + query = f""" + SELECT "{column_name}" FROM "{self.table_name}" + WHERE "{column_name}" IS NOT NULL + GROUP BY "{column_name}" + ORDER BY COUNT(*) DESC + LIMIT 1; + """ - print("Calculating mode manually using data...") - self.load_main_df() - col_data = self.get_processed_column_data(column_name, 'mode') - mode = col_data.mode() + mode = fetch_sql(query, True, self.engine) + print("MODE FROM SQL QUERY: ", mode) + + except Exception as e: + print("AHHH SQL QUERY DIDN'T WORK") + print(f"Error fetching the mode for table {self.table_name} at column {column_name}: {e}") + + mode = None return mode @@ -426,20 +357,88 @@ def _calculate_error_count_dict(self, column_name): :return: The error count dict ({"missing": 10, "mismatch": 5, ...}) """ try: - query = f'SELECT {column_name}, COUNT(*) as cnt FROM {self.table_name} GROUP BY {column_name} ORDER BY cnt DESC' - category_counts = dict(fetch_sql(query, True, self.engine)) + query = f""" + SELECT error_type, COUNT(*) + FROM "{self.error_table_name}" + WHERE column_id = :column_name + GROUP BY error_type + """ + error_counts = dict(fetch_sql(query, False, self.engine, params={'column_name': column_name})) except Exception as e: print(f"Error fetching the error counts for table {self.table_name} at column {column_name}: {e}") + error_counts = None + + + if error_counts is not None: + error_counts = json.dumps(error_counts) + + return error_counts + + # TODO: implement this later + # # Dict mapping from class to error types to error counts + # def _calculate_class_error_count_dict(self, column_name): + # """ + # :param column_name: Name of the column for which the class error count is being calculated + # :return: The class error count dict ({"Male": {"missing": 10, "mismatch": 5, ...}, "Female": {"missing": 10, "mismatch": 5, ...}}) + # """ + # + # try: + # # TODO: check if this works + # ry: + # query = f''' + # SELECT "column_id", "error_type", COUNT(*) AS error_count + # FROM "{self.error_table_name}" + # WHERE "column_id" = :column_name + # GROUP BY "column_id", "error_type" + # ''' + # + # rows = fetch_sql(query, True, self.engine, params={"column_name": column_name}) + # + # counts_by_column = {} + # for r in (rows or []): + # row = dict(r) + # category = row[category_col] + # error_type = row["error_type"] + # count = row["error_count"] + # counts_by_column.setdefault(category, {})[error_type] = count + # + # except Exception as e: + # print(f"Error fetching the error counts for table {self.table_name} at column {column_name}: {e}") + # counts_by_column = None + # if counts_by_column is not None: + # counts_by_column = json.dumps(counts_by_column) + # + # return counts_by_column + + def _calculate_category_count_dict(self, column_name): + """ + :param column_name: Name of the column for which the class error count is being calculated + :return: The class error count dict ({"Male": {"missing": 10, "mismatch": 5, ...}, "Female": {"missing": 10, "mismatch": 5, ...}}) + """ - print("Calculating error counts manually using data...") - self.load_error_df() + try: + query = f""" + SELECT "{column_name}", COUNT(*) + FROM "{self.table_name}" + GROUP BY "{column_name}" + """ + rows = fetch_sql(query, False ,self.engine) category_counts = {} - if not self._error_df.empty: # If error_df is empty (no errors in data selection) - category_counts = self._error_df["error_type"].value_counts().to_dict() + # Put the results into a dict + for (category, count) in rows: + category_counts[category] = count + print("CATEGORY COUNTS DICT", category_counts) + + + except Exception as e: + + print(f"Error fetching the category counts for table {self.table_name} at column {column_name}: {e}") + category_counts = None if category_counts is not None: + # Put the dict into a string so we can actually put it into a SQL table category_counts = json.dumps(category_counts) return category_counts @@ -488,6 +487,7 @@ def get_column_names(self): all_cols = [list(self.col_types.numeric_cols), list(self.col_types.pure_categorical), list(self.col_types.categorical_mixed)] + return all_cols diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index 3b8fc20..b410e55 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -100,6 +100,11 @@ def update_data_profile_table(table_name: str, error_df: pd.DataFrame, columns_s print("COL NAMES", columns_selected_for_wrangling) updated_df = create_data_profile_df(table_name, engine, col_names=columns_selected_for_wrangling, error_df=error_df) + # Can't use db_operations.data_profile because this function is also used for updating preview tables, + # meaning that the "main_table" that this function uses may be a preview table. Using the db_operations data_profile + # has the table name set as the main table and it'll be calculating statistics on the wrong table. So we create a new data + # profile object + data_profile = DataProfile(table_name, engine) key_column = "column_name" diff --git a/app/server_utils/data_attribute_summary_integration.py b/app/server_utils/data_attribute_summary_integration.py index 402a6ff..f017791 100644 --- a/app/server_utils/data_attribute_summary_integration.py +++ b/app/server_utils/data_attribute_summary_integration.py @@ -9,7 +9,7 @@ from app.db_utils.execute_sql import fetch_sql from app.server_utils.service_helpers import get_error_dist, is_categorical, _validate_identifier from app.db_utils.data_profile import DataProfile - +import pandas as pd def get_default_attributes_from_rankings(tablename, engine): """ @@ -45,28 +45,30 @@ def generate_complete_json(tablename): data_profile = DataProfile(tablename, engine) - error_df = data_profile.get_error_df() - main_df = data_profile.get_main_df() + # TODO: get rid of these arguments from the get_error_dist + error_df = pd.read_sql_query(f'SELECT * FROM "{"errors_" + tablename}"', engine) + main_df = pd.read_sql_query(f'SELECT * FROM "{tablename}"', engine) + error_list = get_error_dist(error_df, main_df).to_dict('records') default_attributes = get_default_attributes_from_rankings(tablename, engine) - return { "columnErrors": convert_error_list_to_dict(error_list), - "attributes": list(data_profile._main_df.columns), - "attributeDistributions": build_attribute_distributions(data_profile), + "attributes": list(data_profile.get_col_names()), + "attributeDistributions": build_attribute_distributions(data_profile, main_df), "defaultAttributes": default_attributes } -def get_attribute_stats(data_profile, column): +def get_attribute_stats(data_profile, column, main_df): """ Get statistics for a specific attribute in the DataFrame :param data_profile: Data profile class instance (used for calculating summary stats) :param column: name of the column to get statistics for :return: dictionary containing statistics for the column """ - if is_categorical(data_profile._main_df[column]): + + if data_profile.is_categorical_col(column): return get_categorical_stats(data_profile, column) return get_numeric_stats(data_profile, column) @@ -74,12 +76,13 @@ def build_attribute_distributions(data_profile): """ Build distributions for each attribute in the main DataFrame :param data_profile: Data profile class instance (used for calculating summary stats) + :param main_df: The main DataFrame containing the data :return: dictionary containing distributions for each attribute """ distributions = {} for col in data_profile.get_col_names(): - distributions[col] = get_attribute_stats(data_profile, col) + distributions[col] = get_attribute_stats(data_profile, col, main_df) return distributions def get_categorical_stats(data_profile, column): diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index 6503b59..86617bf 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -189,7 +189,6 @@ def create_data_profile_df(table_name, engine, col_names=None, error_df=None, ma """ print("CREATED DATA_PROFILE DF FOR TABLE", table_name) - data_profile = DataProfile(table_name, engine=engine, main_df=main_df, error_df=error_df) col_attribute_list = [] From 62f84e452e26656ac20928bcb2d7bd25a52f0908 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 10 Jul 2026 17:54:04 -0600 Subject: [PATCH 32/81] Added main_df argument to build_attribute_distributions --- app/server_utils/data_attribute_summary_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server_utils/data_attribute_summary_integration.py b/app/server_utils/data_attribute_summary_integration.py index f017791..e9ea92c 100644 --- a/app/server_utils/data_attribute_summary_integration.py +++ b/app/server_utils/data_attribute_summary_integration.py @@ -72,7 +72,7 @@ def get_attribute_stats(data_profile, column, main_df): return get_categorical_stats(data_profile, column) return get_numeric_stats(data_profile, column) -def build_attribute_distributions(data_profile): +def build_attribute_distributions(data_profile, main_df): """ Build distributions for each attribute in the main DataFrame :param data_profile: Data profile class instance (used for calculating summary stats) From b0e2ac17940f964ad66b7714b7e0b0ac954ddd48 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 10 Jul 2026 17:55:21 -0600 Subject: [PATCH 33/81] Cleaned up data_profile.py and fixed SQL queries not working for _calculate_error_count_dict and _calculate_category_count_dict --- app/db_utils/data_profile.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index b5cd646..bb2a944 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -125,7 +125,7 @@ class DataProfile: This class prioritizes SQL queries and when those don't work, the table is loaded and the pd dataframe method is used instead. """ - def __init__(self, table_name, engine=None, main_df=None, error_df=None): + def __init__(self, table_name, engine): """ :param table_name: name of the main data table :param engine: @@ -141,7 +141,6 @@ def __init__(self, table_name, engine=None, main_df=None, error_df=None): self.engine = engine # IMPORTANT: main_df and error_df are NOT guaranteed to not be None (so that they don't have to be loaded each time for efficiency) # Use get_main_df and get_error_df instead of accessing them directly - self.default_attributes = ['mean', 'median', 'min', 'max', 'n_categories', 'mode', 'error_counts', 'class_error_counts'] self.name_to_func = { 'mean': self._calculate_mean, 'median': self._calculate_median, @@ -150,13 +149,13 @@ def __init__(self, table_name, engine=None, main_df=None, error_df=None): 'n_categories': self._calculate_num_categories, 'mode': self._calculate_mode, 'error_counts': self._calculate_error_count_dict, - 'class_error_counts': self._calculate_class_error_count_dict, + 'category_counts': self._calculate_category_count_dict, } self.attribute_type_assignment = { - 'numeric': ['mean', 'median', 'min', 'max', 'err', 'error_counts'], + 'numeric': ['mean', 'median', 'min', 'max', 'error_counts'], 'categorical': ['n_categories', - 'mode', 'error_counts', 'class_error_counts'], + 'mode', 'error_counts', 'category_counts'], } @@ -198,9 +197,8 @@ def look_up_stat_from_profile(self, attribute_name, column_name): :param column_name: Name of the column for which the statistic is being looked up. :return: The value of the statistic if found, otherwise None. """ - data_profile_table_name = "dp_" + self.table_name try: - query = f'SELECT "{attribute_name}" FROM "{data_profile_table_name}" WHERE "column_name" = :column_name' + query = f'SELECT "{attribute_name}" FROM "{self.data_profile_table_name}" WHERE "column_name" = :column_name' params = {"column_name": column_name} stat = fetch_sql(query, True, self.engine, params) @@ -232,12 +230,14 @@ def calculate_column_attribute(self, attribute_name, column_name, look_up_stat=T if look_up_value is not None: print("Successfully found col attribute in data profile table!") + return to_scalar(look_up_value) calculate_attribute_func = self.name_to_func[attribute_name] + # TODO: save stat off to table if newly calculated + return to_scalar(calculate_attribute_func(column_name)) - # TODO: save stat off to table if newly calculated def _calculate_mean(self, column_name): """ :param column_name: Name of the column for which the mean is being calculated From 380e97adef7f7c291175fd18b8a9de9ac8bc8d4f Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 10 Jul 2026 17:58:14 -0600 Subject: [PATCH 34/81] Refactored run_detectors to create_error_df --- app/routes/routes.py | 4 ++-- app/routes/wrangler_routes_sql.py | 9 ++++----- app/server_utils/service_helpers.py | 3 ++- tests/unit/test_service_helpers.py | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/routes/routes.py b/app/routes/routes.py index 53384cf..3cedda7 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -9,7 +9,7 @@ from app import db_operations, engine from app.server_utils.service_helpers import ( generate_table_name, - run_detectors, + create_error_df, get_sqlalchemy_dtype_map, calculate_attribute_rankings, get_pgraph_redo, get_pgraph_undo, init_pgraph_for_session, create_data_profile_df, ) @@ -34,7 +34,7 @@ def load_file(csv_file, filename): # run the detectors on the uploaded file for the starting data state table_with_id_added = set_id_column(dataframe) start_time = time.time() - detected_data = run_detectors(dataframe) + detected_data = create_error_df(dataframe) time_to_detect = time.time() - start_time app.original_table_name = filename table_name = generate_table_name(filename) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index b410e55..60d3a18 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -7,9 +7,8 @@ from app import engine import traceback import pandas as pd -from app.server_utils.service_helpers import run_detectors, create_previews_1d, create_previews_2d, \ - execute_wrangle_preview, _safe_pg_name, create_data_profile_df -from sqlalchemy import text as sa_text +from app.server_utils.service_helpers import create_error_df, create_previews_1d, create_previews_2d, \ + execute_wrangle_preview, _safe_pg_name, create_data_profile_df, get_sqlalchemy_dtype_map from sqlalchemy import inspect, text @@ -66,10 +65,10 @@ def update_errors_table(table_name: str, columns_selected_for_wrangling: list) - try: df = pd.read_sql_query(f'SELECT * FROM "{table_name}"', engine) - # TODO: optimize this so it doesn't load the whole table into a df first df = df[columns_selected_for_wrangling] - detected_errors_df = run_detectors(df) + # TODO: optimize this so it doesn't load the whole table into a df first + detected_errors_df = create_error_df(df) errors_table_name = f"errors_{table_name}" key_column = "column_id" diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index 86617bf..0a1069c 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -161,7 +161,8 @@ def perform_melt(dfs): return df_combined -def run_detectors(data_frame): +# Previously called run_detectors +def create_error_df(data_frame): """ Runs all 4 detectors that are implemented on the server, on the data, and returns a compiled dataframe of the complete errors diff --git a/tests/unit/test_service_helpers.py b/tests/unit/test_service_helpers.py index 32715a8..397b74b 100644 --- a/tests/unit/test_service_helpers.py +++ b/tests/unit/test_service_helpers.py @@ -3,7 +3,7 @@ import pandas as pd from numpy.ma.testutils import assert_equal -from app.server_utils.service_helpers import clean_table_name, get_whole_table_query, run_detectors, create_error_dict, \ +from app.server_utils.service_helpers import clean_table_name, get_whole_table_query, create_error_df, create_error_dict, \ get_range_of_ids_query, is_categorical, create_bins_for_a_numeric_column, get_2d_bins, \ group_by_attribute, get_error_dist from wranglers.remove_data import remove_data @@ -38,7 +38,7 @@ def test_whole_table_query(self): def test_run_all_detectors_stackoverflow(self): stackoverflow_df = pd.read_csv('../../provided_datasets/stackoverflow_db_uncleaned.csv') - actual_error_df = run_detectors(stackoverflow_df) + actual_error_df = create_error_df(stackoverflow_df) # expected_error_map = {"Age": {3: ["incomplete"], 4: ["mismatch", "incomplete"], 5: ["mismatch", "incomplete"], # 105: ["incomplete"], 159: ["incomplete"]}, # "Continent": {8: ["missing"], 9: ["missing"], 10: ["missing"], @@ -80,14 +80,14 @@ def test_run_all_detectors_stackoverflow(self): def test_run_all_detectors_complaints(self): stackoverflow_df = pd.read_csv('../../provided_datasets/complaints-2025-04-21_17_31.csv') - actual_error_df = run_detectors(stackoverflow_df) + actual_error_df = create_error_df(stackoverflow_df) self.assertEqual(True,True) def test_create_error_dictionary(self): # stackoverflow_df = pd.read_csv('../provided_datasets/stackoverflow_db_uncleaned.csv') stackoverflow_df = pd.read_csv('../../provided_datasets/stackoverflow_db_uncleaned.csv') - res_df = run_detectors(stackoverflow_df) + res_df = create_error_df(stackoverflow_df) create_error_dict(res_df,200) def test_get_range_of_ids_query(self): @@ -136,6 +136,6 @@ def test_group_by_categorical_group(self): def test_get_error_dis(self): stackoverflow_df = pd.read_csv('../../provided_datasets/stackoverflow_db_uncleaned.csv') - error_table = run_detectors(stackoverflow_df) + error_table = create_error_df(stackoverflow_df) error_dist = get_error_dist(error_table,stackoverflow_df) assert_equal(1,1) From 3797e0b35f6bebdbebc8e887583ba25167c673d5 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 10 Jul 2026 18:05:13 -0600 Subject: [PATCH 35/81] Fixed bug relating to dtypes not matching up when trying to update tables with staged tables of updated columns --- app/routes/routes.py | 8 ++++---- app/routes/wrangler_routes_sql.py | 26 +++++++++++++++----------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/app/routes/routes.py b/app/routes/routes.py index 3cedda7..78968fb 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -54,16 +54,16 @@ def load_file(csv_file, filename): table_with_id_added.to_sql(table_name_with_node_id, engine, if_exists='replace', dtype=dtype_map) detected_data.to_sql("errors_" + table_name_with_node_id, engine, if_exists='replace') - data_profile_df = create_data_profile_df(table_name_with_node_id, engine, error_df=detected_data, main_df=dataframe) - - data_profile_df.to_sql("dp_" + table_name_with_node_id, engine, if_exists='replace') + db_operations.load_table(table_name_with_node_id) + data_profile_df = create_data_profile_df(db_operations.data_profile) + dtype_map = db_operations.data_profile.dtype_dict + data_profile_df.to_sql("dp_" + table_name_with_node_id, engine, if_exists='replace', dtype=dtype_map) """ now we fully init the DBOperations object that was first initialized in init.py, get the actual row counts since .to_sql is buggy and not right """ - db_operations.load_table(table_name_with_node_id) rows_affected = db_operations.get_row_count(table_name_with_node_id) detected_rows_affected = db_operations.get_row_count("errors_" + table_name_with_node_id) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index 60d3a18..3006d3e 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -11,12 +11,19 @@ execute_wrangle_preview, _safe_pg_name, create_data_profile_df, get_sqlalchemy_dtype_map from sqlalchemy import inspect, text - +from app.db_utils.data_profile import DataProfile """ Wrangling Endpoints - In-place modification of tables """ +def get_table_dtypes(target_table_name, engine): + """Build a dtype dict for to_sql() by reflecting the target table's real column types.""" + inspector = inspect(engine) + columns = inspector.get_columns(target_table_name) + # col["type"] is already a SQLAlchemy type instance we can hand straight to to_sql + return {col["name"]: col["type"] for col in columns} + # Where updated_df is just the data that needed to actually be updated # Assumes that updated_df has the same columns as the target table # TODO: reimplement with "dirty flags" @@ -27,12 +34,12 @@ def update_table(updated_df, target_table_name, key_col, cols_to_remove): {"categories": cols_to_remove} ) - staging_table = _safe_pg_name(target_table_name, "_staging") + staging_table_name = _safe_pg_name(target_table_name, "_staging") - cols_to_update = updated_df.columns + dtype_dict = get_table_dtypes(target_table_name, engine) # 1. Push data to a temp staging table - updated_df.to_sql(staging_table, engine, if_exists='replace', index=False) + updated_df.to_sql(staging_table_name, engine, if_exists='replace', dtype=dtype_dict) # Make sure that there's a main errors table we can update inspector = inspect(engine) @@ -40,16 +47,13 @@ def update_table(updated_df, target_table_name, key_col, cols_to_remove): # 2. Set-based update, Postgres native syntax with engine.begin() as conn: - set_clause = ", ".join(f'"{c}" = staged."{c}"' for c in cols_to_update) conn.execute(text(f''' - UPDATE "{target_table_name}" target - SET {set_clause} - FROM "{staging_table}" staged - WHERE target."{key_col}" = staged."{key_col}" + INSERT INTO "{target_table_name}" + SELECT * + FROM "{staging_table_name}" ''')) - conn.execute(text(f'DROP TABLE "{staging_table}"')) - + conn.execute(text(f'DROP TABLE "{staging_table_name}"')) # ───────────────────────────────────────────────────────────────────────────── # Helper: Re-run error detection after modification From 5f050a5a0a135788baa398ae98abf853f2354a2b Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 10 Jul 2026 18:07:41 -0600 Subject: [PATCH 36/81] Cleaned up create_error_df and create_data_profile_df --- app/server_utils/service_helpers.py | 69 +++++++++++++++++------------ 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index 0a1069c..4bf8108 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -170,51 +170,63 @@ def create_error_df(data_frame): :return: a single compiled dataframe of all the errors detected """ df_with_id = set_id_column(data_frame) + + # TODO: optimize these functions anomaly_df = pd.DataFrame(anomaly(df_with_id.copy())).rename_axis("ID", axis="index").reset_index() incomplete_df = pd.DataFrame(incomplete(df_with_id.copy())).rename_axis("ID", axis="index").reset_index() missing_value_df = pd.DataFrame(missing_value(df_with_id.copy())).rename_axis("ID", axis="index").reset_index() datatype_mismatch_df = pd.DataFrame(datatype_mismatch(df_with_id.copy())).rename_axis("ID", axis="index").reset_index() frames = [anomaly_df, incomplete_df, missing_value_df,datatype_mismatch_df] - return perform_melt(frames) + + df = perform_melt(frames) + print("CREATE ERROR TABLE TYPE MAP", df.dtypes) + return df # TODO: CLEAN UP THIS FUNCTION # TODO: Test this function!!! (Write test for it) -def create_data_profile_df(table_name, engine, col_names=None, error_df=None, main_df=None): +def create_data_profile_df(data_profile, col_names=None): """ - :param table_name: the name of the table in the database - :param engine: the engine to use + :param data_profile: the data profile object :param col_names: the column names of interest in the table - :param error_df: the error dataframe (optional) - :param main_df: the main dataframe (optional) :return: a dataframe of the data profile for the table """ - print("CREATED DATA_PROFILE DF FOR TABLE", table_name) + print("CREATED DATA_PROFILE DF FOR TABLE", data_profile.table_name) + # Dict of attributes that will be in the data profile and the type that they should be + default_attributes = ['mean', 'median', 'min', 'max', 'n_categories', + 'mode', 'error_counts', + 'category_counts'] - col_attribute_list = [] + col_list = [] + + # If col_names is not provided (no specific columns to create a dp for), use all column names from the data profile if col_names is None: col_names = data_profile.get_col_names() for col in col_names: - if col not in data_profile.get_col_names(): - continue row_dict = {'column_name': col} - for attribute in data_profile.default_attributes: - if attribute not in data_profile.attribute_type_assignment['categorical'] and attribute not in data_profile.attribute_type_assignment['numeric']: - # Attribute doesn't exist in either categorical and numeric - print(f"ERROR: INVALID ATTRIBUTE {attribute}") - print("Skipping this attribute") + for attribute in default_attributes: + + # Make sure that attribute and the column type match + + numeric = (data_profile.is_numeric_col(col) and attribute in data_profile.attribute_type_assignment['numeric']) + categorical = (data_profile.is_categorical_col(col) and attribute in data_profile.attribute_type_assignment['categorical']) + + if not (numeric or categorical): + + row_dict[attribute] = None + continue print("CALCULATING ATTRIBUTE: ", attribute) print("COLUMN: ", col) row_dict[attribute] = data_profile.calculate_column_attribute(attribute, col, False) - col_attribute_list.append(row_dict) + col_list.append(row_dict) - df = pd.DataFrame(col_attribute_list) + df = pd.DataFrame(col_list) return df @@ -428,7 +440,7 @@ def execute_wrangle_preview(table, preview_table, preview_name_fn, db_operations # new_table_name = pgraph_entry_point(table, preview_table_trimmed, wrangle_executed) new_table_name = n_wrangle(table, preview_table_trimmed, wrangle_executed) app.db_operations.rename_preview_to_new(preview_table, new_table_name) - db_operations.load_table(new_table_name, f"errors_{new_table_name}") + db_operations.load_table(new_table_name, f"errors_{new_table_name}", f"dp_{new_table_name}") app.db_operations.update_rankings(new_table_name) @@ -521,10 +533,10 @@ def create_previews_1d(table, row_ids, cols, safe_pg_name_fn, update_errors_fn, query.remove_rows_by_ids(table=preview_delete_table_name, ids=row_ids) query.impute_by_ids(table=preview_impute_table_name, col=cols[0], ids=row_ids) - errors_df_delete = update_errors_fn(preview_delete_table_name, cols) - errors_df_impute = update_errors_fn(preview_impute_table_name, cols) - update_data_profile_table_fn(preview_delete_table_name, errors_df_delete, cols) - update_data_profile_table_fn(preview_impute_table_name, errors_df_impute, cols) + update_errors_fn(preview_delete_table_name, cols) + update_errors_fn(preview_impute_table_name, cols) + update_data_profile_table_fn(preview_delete_table_name, cols) + update_data_profile_table_fn(preview_impute_table_name, cols) return { "success": True, @@ -563,12 +575,13 @@ def create_previews_2d(table, row_ids, cols, safe_pg_name_fn, update_errors_fn, query.impute_by_ids(table=preview_impute_x_table_name, col=cols[0], ids=row_ids) query.impute_by_ids(table=preview_impute_y_table_name, col=cols[1], ids=row_ids) - errors_df_delete = update_errors_fn(preview_delete_table_name, cols) - errors_df_impute_x = update_errors_fn(preview_impute_x_table_name, cols) - errors_df_impute_y = update_errors_fn(preview_impute_y_table_name, cols) - update_data_profile_table_fn(preview_delete_table_name, errors_df_delete, cols) - update_data_profile_table_fn(preview_impute_x_table_name, errors_df_impute_x, cols) - update_data_profile_table_fn(preview_impute_y_table_name, errors_df_impute_y, cols) + update_errors_fn(preview_delete_table_name, cols) + update_errors_fn(preview_impute_x_table_name, cols) + update_errors_fn(preview_impute_y_table_name, cols) + + update_data_profile_table_fn(preview_delete_table_name, cols) + update_data_profile_table_fn(preview_impute_x_table_name, cols) + update_data_profile_table_fn(preview_impute_y_table_name, cols) return { From 47ec3e96ff7e0daa5fa008c0de71b8cec6f88b2e Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 10 Jul 2026 18:08:04 -0600 Subject: [PATCH 37/81] Changed argument name of execute_wrangle_preview to be more clear --- app/server_utils/service_helpers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index 4bf8108..446c5b2 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -414,7 +414,7 @@ def create_bins_for_a_numeric_column(column,bin_count): return pd.cut(column_numeric, bins=bin_count) -def execute_wrangle_preview(table, preview_table, preview_name_fn, db_operations): +def execute_wrangle_preview(table, preview_table, safe_pg_name_fn, db_operations): """ Promote a preview table to the new current table and make it as a new node in the pgraph 1. Drop all other preview tables (and their errors_ siblings) @@ -425,10 +425,10 @@ def execute_wrangle_preview(table, preview_table, preview_name_fn, db_operations # from app import engine, db_operations all_possible_previews = [ - preview_name_fn(table, "_preview_delete"), - preview_name_fn(table, "_preview_impute"), - preview_name_fn(table, "_preview_impute_x"), - preview_name_fn(table, "_preview_impute_y"), + safe_pg_name_fn(table, "_preview_delete"), + safe_pg_name_fn(table, "_preview_impute"), + safe_pg_name_fn(table, "_preview_impute_x"), + safe_pg_name_fn(table, "_preview_impute_y"), ] app.db_operations.drop_preview_tables(all_possible_previews, preview_table) From 317abca86dde78e45d27bcc6ccbfd3e9845b10c7 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 10 Jul 2026 18:08:47 -0600 Subject: [PATCH 38/81] Cleaned up update_data_profile_table --- app/routes/wrangler_routes_sql.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index 3006d3e..511f4c8 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -95,26 +95,25 @@ def update_errors_table(table_name: str, columns_selected_for_wrangling: list) - # TODO: Make update_data_profile_table and update_errors_table more similar # TODO: Re-implement with "dirty flags" -def update_data_profile_table(table_name: str, error_df: pd.DataFrame, columns_selected_for_wrangling: list) -> None: +# TODO:optimize this so it doesn't load the whole table into a df first +def update_data_profile_table(table_name: str, columns_selected_for_wrangling: list) -> None: try: - # TODO: optimize this so it doesn't load the whole table into a df first dp_table_name = f"dp_{table_name}" - print("COL NAMES", columns_selected_for_wrangling) - updated_df = create_data_profile_df(table_name, engine, col_names=columns_selected_for_wrangling, error_df=error_df) # Can't use db_operations.data_profile because this function is also used for updating preview tables, # meaning that the "main_table" that this function uses may be a preview table. Using the db_operations data_profile # has the table name set as the main table and it'll be calculating statistics on the wrong table. So we create a new data # profile object data_profile = DataProfile(table_name, engine) + updated_df = create_data_profile_df(data_profile, col_names=columns_selected_for_wrangling) + key_column = "column_name" update_table(updated_df, dp_table_name, key_column, columns_selected_for_wrangling) - print(f"✓ Updated data profile table: {dp_table_name}") except Exception as e: print(f"ERROR: Could not update data profile table for {table_name}: {e}") From d21cf8d1b7a8eb1fcfe235ce9c31442ffda7e6f2 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 10 Jul 2026 18:09:18 -0600 Subject: [PATCH 39/81] Cleaned up wrangle_delete_column --- app/routes/wrangler_routes_sql.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index 511f4c8..de462de 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -113,7 +113,6 @@ def update_data_profile_table(table_name: str, columns_selected_for_wrangling: l update_table(updated_df, dp_table_name, key_column, columns_selected_for_wrangling) - print(f"✓ Updated data profile table: {dp_table_name}") except Exception as e: print(f"ERROR: Could not update data profile table for {table_name}: {e}") @@ -232,16 +231,17 @@ def wrangle_delete_column(): """ try: body = request.get_json(force=True) - table = db_operations.main_table_name + table_name = db_operations.main_table_name column = body["column"] - print(f"Deleting column '{column}' from table '{table}'") + print(f"Deleting column '{column}' from table '{table_name}'") # Delete the column - remaining_columns = query.delete_column(table=table, column=column) + remaining_columns = query.delete_column(table=table_name, column=column) # Re-run error detection - update_errors_table(table) + update_errors_table(table_name, [column]) + update_data_profile_table(table_name, [column]) return { "success": True, From 3daa10001dcd71199ae7bc09100fb803b05fca0e Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Thu, 16 Jul 2026 18:44:01 -0600 Subject: [PATCH 40/81] Created get_table_dtypes function to fix error with update_table where we can't update tables because the column data types don't match --- app/db_utils/query.py | 8 +++++++- app/routes/wrangler_routes_sql.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/db_utils/query.py b/app/db_utils/query.py index 0069b5a..552eda9 100644 --- a/app/db_utils/query.py +++ b/app/db_utils/query.py @@ -4,7 +4,7 @@ from typing import Dict, Any, List, Tuple from sqlalchemy import text, Engine from app import engine - +from sqlalchemy import inspect # ───────────────────────────────────────────────────────────────────────────── # Helper Functions @@ -15,6 +15,12 @@ "decimal", "numeric", "real", "double precision" } +def get_table_dtypes(target_table_name, engine): + """Build a dtype dict for to_sql() by reflecting the target table's real column types.""" + inspector = inspect(engine) + columns = inspector.get_columns(target_table_name) + # col["type"] is already a SQLAlchemy type instance we can hand straight to to_sql + return {col["name"]: col["type"] for col in columns} def _is_numeric(conn, col: str, table_name: str) -> bool: """Check if a column is numeric.""" diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index de462de..5e5fe8b 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -36,7 +36,7 @@ def update_table(updated_df, target_table_name, key_col, cols_to_remove): staging_table_name = _safe_pg_name(target_table_name, "_staging") - dtype_dict = get_table_dtypes(target_table_name, engine) + dtype_dict = query.get_table_dtypes(target_table_name, engine) # 1. Push data to a temp staging table updated_df.to_sql(staging_table_name, engine, if_exists='replace', dtype=dtype_dict) From 9f5b591c6e6b5fa61f57b0aa1ad4d442350906d0 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Thu, 16 Jul 2026 18:48:36 -0600 Subject: [PATCH 41/81] Changed ColumnTypes so numeric categories are columns with majority numeric and categorical are columns with majority categorical. Previous implementation in Column Types required all to be numeric / categorical. --- app/db_utils/data_profile.py | 65 +++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index bb2a944..7052f32 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -29,13 +29,20 @@ def to_scalar(val): class ColumnTypes: def __init__(self, main_table_name: str, engine): + # Cols where majority of the rows are numeric self.numeric_cols = set() - self.categorical_mixed = set() - self.pure_categorical = set() + self.mixed_cols = set() + # Cols where majority of the rows are categorical + self.categorical_cols = set() self.engine = engine + + self.numeric_types = [ + 'integer', 'bigint', 'numeric', + 'real', 'double precision', 'smallint' + ] self.gather_numeric_cols(main_table_name) self.gather_mixed_cols(main_table_name) - + self.categorize_mixed_cols(main_table_name) def gather_numeric_cols(self, main_table_name: str): """ @@ -43,26 +50,22 @@ def gather_numeric_cols(self, main_table_name: str): :arg: main_table_name: name of the main table. """ - fetch_col_types = f'''SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '{main_table_name}';''' fetched_rows = fetch_sql(fetch_col_types, False, self.engine) if fetched_rows: - numeric_types = { - 'integer', 'bigint', 'numeric', - 'real', 'double precision', 'smallint' - } for row in fetched_rows: col_name = row[0] + # This datatype will only be numeric if the whole column is numeric data_type = row[1] - if data_type in numeric_types: + if data_type in self.numeric_types: self.numeric_cols.add(col_name) else: - self.categorical_mixed.add(col_name) + self.mixed_cols.add(col_name) else: raise Exception(f"No rows fetched from table: {main_table_name}") @@ -74,7 +77,7 @@ def gather_mixed_cols(self, main_table_name: str): """ # There are no categorical columns in the dataset. - if len(self.categorical_mixed) == 0: + if len(self.mixed_cols) == 0: return numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" @@ -83,23 +86,45 @@ def gather_mixed_cols(self, main_table_name: str): # Stop early if a mixed type is found, since that makes the entire column of mixed type. queries = [ f"""( - SELECT '{col}' AS column_name - FROM "{main_table_name}" - WHERE "{col}" ~ {numeric_regex} - LIMIT 1 + SELECT '{col}' AS column_name + FROM "{main_table_name}" + WHERE pg_input_is_valid("{col}", 'numeric') + LIMIT 1 )""" - for col in self.categorical_mixed + for col in self.mixed_cols ] fetch_mixed_types = "\nUNION ALL\n".join(queries) mixed_cols = fetch_sql(fetch_mixed_types, False, self.engine) mixed_col_names = set(row[0] for row in mixed_cols) if mixed_cols else set() - self.pure_categorical = self.categorical_mixed - mixed_col_names - self.categorical_mixed = mixed_col_names + self.categorical_cols = self.mixed_cols - mixed_col_names + self.mixed_cols = mixed_col_names + + def categorize_mixed_cols(self, main_table_name: str): + """ + Categorizes the mixed columns into numeric and categorical based on the majority of their values. + :arg: main_table_name: name of the main table. + """ + for col in self.mixed_cols: + query = f""" + SELECT + SUM(CASE WHEN pg_input_is_valid("{col}", \'numeric\' )THEN 1 ELSE 0 END) AS numeric_count, + COUNT(*) AS total_count + FROM "{main_table_name}"; + """ + result = fetch_sql(query, False, self.engine) + if result: + numeric_count, total_count = result[0] + if numeric_count > total_count / 2: + self.numeric_cols.add(col) + else: + self.categorical_cols.add(col) + else: + raise Exception(f"No rows fetched for column: {col} in table: {main_table_name}") def is_categorical_col(self, col_name: str): - return col_name in self.pure_categorical + return col_name in self.categorical_cols def is_numeric_col(self, col_name: str): """ @@ -116,7 +141,7 @@ def is_mixed_col(self, col_name: str): :arg: col_name: name of the column (assumes it is from the same table used to construct this class). :return: whether the given col_name is of mixed type. """ - return col_name in self.categorical_mixed + return col_name in self.mixed_cols class DataProfile: """ From ec9a06da083d72cae9460754b5897724c31ec061 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Thu, 16 Jul 2026 18:51:51 -0600 Subject: [PATCH 42/81] Made it so numeric summary stats work even with mixed numeric columns (even just having one string in the column turns the whole column into a text column) --- app/db_utils/data_profile.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index 7052f32..cb5c664 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -239,7 +239,18 @@ def calculate_summary_stat_using_sql(self, stat_query, column_name): :param column_name: Name of the column for which the statistic is being looked up. :return: The value of the statistic if found, otherwise None. """ - query = f'SELECT {stat_query}("{column_name}") FROM "{self.table_name}"' + + # Casting to numeric values just in case there is a data type mismatch and the single string variable + # Turns an entire numeric column into text + query = (f'SELECT {stat_query}' + f'("{column_name}"::numeric) FROM ' + f'"{self.table_name}"') + + # If its a numeric column with at least one string / categorical value, we only keep the numeric values so we + # Can properly do calculations + if self.is_mixed_col(column_name): + query += f' WHERE pg_input_is_valid("{column_name}", \'numeric\')' + stat = fetch_sql(query, True, self.engine) return stat @@ -288,11 +299,16 @@ def _calculate_median(self, column_name): # Try get the median from SQL first, if it fails, calculate manually using data frame try: - query = f'SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY "{column_name}") FROM "{self.table_name}"' + query = f'SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY "{column_name}"::numeric) FROM "{self.table_name}"' + + # If its a numeric column with at least one string / categorical value, we only keep the numeric values so we + # Can properly do calculations + if self.is_mixed_col(column_name): + query += f' WHERE pg_input_is_valid("{column_name}", \'numeric\')' median = fetch_sql(query, True, self.engine) except Exception as e: print(f"Error fetching the median for table {self.table_name} at column {column_name}: {e}") - median = None + median = float('nan') return median From a150433cdddfcc009ee01a7678dd797d0b2dc7c8 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Thu, 16 Jul 2026 18:52:35 -0600 Subject: [PATCH 43/81] Fixed bug in _calculate_error_count_dict --- app/db_utils/data_profile.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index cb5c664..8bf79f3 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -186,10 +186,6 @@ def __init__(self, table_name, engine): self.dtype_dict = None - - - - # TODO: Make the sql query for this work def get_col_names(self): """ self: DataProfile instance @@ -405,14 +401,17 @@ def _calculate_error_count_dict(self, column_name): GROUP BY error_type """ error_counts = dict(fetch_sql(query, False, self.engine, params={'column_name': column_name})) + + if error_counts is not None: + error_counts = json.dumps(error_counts) + else: + error_counts = json.dumps({}) except Exception as e: print(f"Error fetching the error counts for table {self.table_name} at column {column_name}: {e}") error_counts = None - if error_counts is not None: - error_counts = json.dumps(error_counts) return error_counts From 6227fa97ab4c10cc6293ce4f806137680b855599 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Thu, 16 Jul 2026 18:53:47 -0600 Subject: [PATCH 44/81] Added functions to get list different column types --- app/db_utils/data_profile.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index 8bf79f3..7182252 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -529,5 +529,13 @@ def get_column_names(self): return all_cols + def get_numeric_cols(self): + return list(self.col_types.numeric_cols) + + def get_categorical_cols(self): + return list(self.col_types.categorical_cols) + + def get_mixed_cols(self): + return list(self.col_types.mixed_cols) From 099cac4826215089bed37c49e3f0cc5a82aa36ff Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Thu, 16 Jul 2026 18:54:16 -0600 Subject: [PATCH 45/81] Refactored categorical_mixed to mixed_cols --- app/db_utils/data_profile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index 7182252..f2de3d4 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -524,8 +524,8 @@ def get_column_names(self): :return: List of column names """ # uses the sets of column names from the ColumnTypes class to get all column names - all_cols = [list(self.col_types.numeric_cols), list(self.col_types.pure_categorical), - list(self.col_types.categorical_mixed)] + all_cols = [list(self.col_types.numeric_cols), list(self.col_types.categorical_cols), + list(self.col_types.mixed_cols)] return all_cols From 6b9682c6c4ac5584bab06db0d50b0d93c74f9064 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Thu, 16 Jul 2026 18:55:35 -0600 Subject: [PATCH 46/81] Created tests for Data Profile functions --- app/server_utils/service_helpers.py | 2 - tests/unit/test_data_profile.py | 107 ++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_data_profile.py diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index 446c5b2..7295c44 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -182,8 +182,6 @@ def create_error_df(data_frame): print("CREATE ERROR TABLE TYPE MAP", df.dtypes) return df -# TODO: CLEAN UP THIS FUNCTION -# TODO: Test this function!!! (Write test for it) def create_data_profile_df(data_profile, col_names=None): """ :param data_profile: the data profile object diff --git a/tests/unit/test_data_profile.py b/tests/unit/test_data_profile.py new file mode 100644 index 0000000..bbdc435 --- /dev/null +++ b/tests/unit/test_data_profile.py @@ -0,0 +1,107 @@ +import unittest +from decimal import Decimal + +import pandas as pd +from app.db_utils.data_profile import DataProfile +from app import engine +import json +from app.db_utils.execute_sql import execute_sql + + +data_profile_df = pd.DataFrame( + { + 'column_name': ['name', 'age', 'city', 'species', 'height', 'hair_color'], + 'mean': [None, Decimal('26.4'), None, None, Decimal('18.2'), None], + 'median': [None, Decimal('30.0'), None, None, Decimal('5.5'), None], + 'min': [None, Decimal('3'), None, None, Decimal('1.2'), None], + 'max': [None, Decimal('60'), None, None, Decimal('6.0'), None], + 'n_categories': [5, None, 4, 2, None, 3], + 'mode': ['Mari', None, 'Phoenix', 'human', None, 'black'], + 'error_counts': [json.dumps({}), json.dumps({}), json.dumps({'missing': 1}), json.dumps({}), json.dumps({'mismatch': 1}), json.dumps({'mismatch': 2})], + 'category_counts': [json.dumps({'Mari':2, 'Seb': 1, 'Zee': 1, 'Juju': 1}), None, json.dumps({'New York': 1, 'Houston': 2, 'Phoenix': 1}), json.dumps({'human': 3, 'dog': 2}), None, json.dumps({'black': 3, 'brown': 1, '2': 1, '1': 1})], + } +) + +main_df = pd.DataFrame( + { + 'name': ['Mari', 'Seb', 'Zee', 'Juju', 'Mari'], + 'age': [30, 60, 3, 9, 30], + 'city': ['New York', 'Houston', None, 'Houston', 'Phoenix'], + 'species': ['human', 'human', 'dog', 'dog', 'human'], + 'height': [5.5, 6.0, 1.2, "one", 5.5], + 'hair_color': ['black', 'brown', 2, 'black', 1], + } +) + +error_df = pd.DataFrame( + { + 'row_id': [3, 4, 3, 5], + 'column_id': ['city','height', 'hair_color', 'hair_color'], + 'error_type': ['missing', 'mismatch', 'mismatch', 'mismatch'], + } +) + +data_profile_table_name = 'dp_main_mari_test' +main_df_table_name = 'main_mari_test' +error_df_table_name = 'errors_main_mari_test' + +main_df.to_sql('main_mari_test', con=engine, if_exists='replace', index=False) +data_profile_df.to_sql('dp_main_mari_test', con=engine, if_exists='replace', index=False) +error_df.to_sql('errors_main_mari_test', con=engine, if_exists='replace', index=False) + + +data_profile = DataProfile('main_mari_test', engine) + +class MyTestCase(unittest.TestCase): + def test_get_col_names(self): + self.assertEqual(data_profile.get_col_names(), ['name', 'age', 'city', 'species', 'height', 'hair_color']) + + def test_look_up_stat_from_profile(self): + self.assertEqual(data_profile.look_up_stat_from_profile('mean', 'age'), 26.4) + self.assertEqual(data_profile.look_up_stat_from_profile('n_categories', 'species'), 2) + self.assertEqual(data_profile.look_up_stat_from_profile('mode', 'species'), 'human') + self.assertEqual(data_profile.look_up_stat_from_profile('mode', 'name'), 'Mari') + self.assertEqual(data_profile.look_up_stat_from_profile('max', 'age'), 60) + self.assertEqual(data_profile.look_up_stat_from_profile('error_counts', 'city'), json.dumps({'missing': 1})) + self.assertEqual(data_profile.look_up_stat_from_profile('category_counts', 'species'), json.dumps({'human': 3, 'dog': 2})) + # Should be None + self.assertEqual(data_profile.look_up_stat_from_profile('category_counts', 'height'), None) + self.assertEqual(data_profile.look_up_stat_from_profile('n_categories', 'age'), None) + self.assertEqual(data_profile.look_up_stat_from_profile('mode', 'age'), None) + + # def test_calculate_column_attribute(self): + # self.assertEqual(data_profile.calculate_column_attribute('mean', 'age'), 26.4) + # self.assertEqual(data_profile.calculate_column_attribute('n_categories', 'species'), 2) + # self.assertEqual(data_profile.calculate_column_attribute('mode', 'species'), 'human') + # self.assertEqual(data_profile.calculate_column_attribute('mode', 'name'), 'Mari') + # self.assertEqual(data_profile.calculate_column_attribute('max', 'age'), 60) + # self.assertEqual(data_profile.calculate_column_attribute('error_counts', 'city'), json.dumps({'missing': 1})) + # self.assertEqual(data_profile.calculate_column_attribute('category_counts', 'species'), json.dumps({'human': 3, 'dog': 2})) + # # Should be None + # self.assertEqual(data_profile.calculate_column_attribute('category_counts', 'height'), None) + # self.assertEqual(data_profile.calculate_column_attribute('n_categories', 'age'), None) + # self.assertEqual(data_profile.calculate_column_attribute('mode', 'age'), None) + + def test_get_mixed_cols(self): + # Height is a mixed column as well as a numeric column because although it is mostly numeric, it has one value + # that is a string + self.assertEqual(set(data_profile.get_mixed_cols()), set(['height', 'hair_color'])) + + def test_get_numeric_cols(self): + self.assertEqual(set(data_profile.get_numeric_cols()), set(['age', 'height'])) + + def test_get_categorical_cols(self): + self.assertEqual(set(data_profile.get_categorical_cols()), set(['name', 'city', 'species', 'hair_color'])) + + def test_calculate_error_count_dict(self): + self.assertEqual(data_profile._calculate_error_count_dict('name'), json.dumps({})) + self.assertEqual(data_profile._calculate_error_count_dict('city'), json.dumps({'missing': 1})) + self.assertEqual(data_profile._calculate_error_count_dict('height'), json.dumps({'mismatch': 1})) + self.assertEqual(data_profile._calculate_error_count_dict('hair_color'), json.dumps({'mismatch': 2})) + + +if __name__ == '__main__': + unittest.main() + execute_sql(f'DROP TABLE IF EXISTS "{main_df_table_name}"', engine) + execute_sql(f'DROP TABLE IF EXISTS "{data_profile_table_name}"', engine) + execute_sql(f'DROP TABLE IF EXISTS "{error_df_table_name}"', engine) From e43b8273289709b1b6b6dbe626ef816123d39ab9 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 19 Jul 2026 13:17:28 -0600 Subject: [PATCH 47/81] Moved ColumnTypes class to its own script --- app/db_utils/column_types.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 app/db_utils/column_types.py diff --git a/app/db_utils/column_types.py b/app/db_utils/column_types.py new file mode 100644 index 0000000..e69de29 From a96f717e3247279163b40b8c50f338bc9ee0b674 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 19 Jul 2026 13:18:01 -0600 Subject: [PATCH 48/81] Moved ColumnTypes class to its own script --- app/db_utils/column_types.py | 135 +++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/app/db_utils/column_types.py b/app/db_utils/column_types.py index e69de29..855b540 100644 --- a/app/db_utils/column_types.py +++ b/app/db_utils/column_types.py @@ -0,0 +1,135 @@ +from app.db_utils.execute_sql import fetch_sql +""" +--- ColumnTypes --- +Inspects a table's schema to classify each column as numeric, categorical, or mixed-type. +""" + +class ColumnTypes: + def __init__(self, main_table_name: str, engine): + # Cols where majority of the rows are numeric + self.numeric_cols = set() + self.mixed_cols = set() + # Cols where majority of the rows are categorical + self.categorical_cols = set() + self.engine = engine + + self.numeric_types = [ + 'integer', 'bigint', 'numeric', + 'real', 'double precision', 'smallint' + ] + self.gather_numeric_cols(main_table_name) + self.gather_mixed_cols(main_table_name) + self.categorize_mixed_cols(main_table_name) + + def get_col_type(self, column_name): + """ + :param column_name: Name of the column for which the type is being checked + :return: The type of the column in a string + """ + if self.is_numeric_col(column_name): + return "numeric" + elif self.is_categorical_col(column_name): + return "categorical" + elif self.is_mixed_col(column_name): + return "mixed" + else: + return None + + def gather_numeric_cols(self, main_table_name: str): + """ + Distinguishes the numeric columns from the categorical columns. + :arg: main_table_name: name of the main table. + """ + + fetch_col_types = f'''SELECT column_name, data_type + FROM information_schema.columns + WHERE table_name = '{main_table_name}';''' + + fetched_rows = fetch_sql(fetch_col_types, False, self.engine) + if fetched_rows: + + for row in fetched_rows: + col_name = row[0] + # This datatype will only be numeric if the whole column is numeric + data_type = row[1] + + if data_type in self.numeric_types: + self.numeric_cols.add(col_name) + else: + self.mixed_cols.add(col_name) + else: + raise Exception(f"No rows fetched from table: {main_table_name}") + + + def gather_mixed_cols(self, main_table_name: str): + """ + Gather the columns that are labeled as categorical but contain numeric data as well. + :arg: main_table_name: name of the main table. + """ + + # There are no categorical columns in the dataset. + if len(self.mixed_cols) == 0: + return + + numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" + + # Initialized in the other constructor func gather_numeric_cols. This starts as all categorical columns. + # Stop early if a mixed type is found, since that makes the entire column of mixed type. + queries = [ + f"""( + SELECT '{col}' AS column_name + FROM "{main_table_name}" + WHERE pg_input_is_valid("{col}", 'numeric') + LIMIT 1 + )""" + for col in self.mixed_cols + ] + + fetch_mixed_types = "\nUNION ALL\n".join(queries) + mixed_cols = fetch_sql(fetch_mixed_types, False, self.engine) + + mixed_col_names = set(row[0] for row in mixed_cols) if mixed_cols else set() + self.categorical_cols = self.mixed_cols - mixed_col_names + self.mixed_cols = mixed_col_names + + def categorize_mixed_cols(self, main_table_name: str): + """ + Categorizes the mixed columns into numeric and categorical based on the majority of their values. + :arg: main_table_name: name of the main table. + """ + for col in self.mixed_cols: + query = f""" + SELECT + SUM(CASE WHEN pg_input_is_valid("{col}", \'numeric\' )THEN 1 ELSE 0 END) AS numeric_count, + COUNT(*) AS total_count + FROM "{main_table_name}"; + """ + result = fetch_sql(query, False, self.engine) + if result: + numeric_count, total_count = result[0] + if numeric_count > total_count / 2: + self.numeric_cols.add(col) + else: + self.categorical_cols.add(col) + else: + raise Exception(f"No rows fetched for column: {col} in table: {main_table_name}") + + def is_categorical_col(self, col_name: str): + return col_name in self.categorical_cols + + def is_numeric_col(self, col_name: str): + """ + Determines whether the given column from the table used to construct this class is numeric. + :arg: col_name: name of the column (assumes it is from the same table used to construct this class). + :return: whether the given col_name is numeric. + """ + return col_name in self.numeric_cols + + + def is_mixed_col(self, col_name: str): + """ + Determines whether the given column from the table used to construct this class is of mixed type. + :arg: col_name: name of the column (assumes it is from the same table used to construct this class). + :return: whether the given col_name is of mixed type. + """ + return col_name in self.mixed_cols From 2ec0c8badfd2890d8b07d65f7e3321e553298886 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 19 Jul 2026 13:18:07 -0600 Subject: [PATCH 49/81] Revert "Removed ColumnTypes from db_functions_sql.py and added data profile instance into DBOperations to get column type functions" This reverts commit 51786a37ca6b665caf31de7796e4020ce5fa1366. --- app/db_utils/db_functions_sql.py | 138 +++++++++++++++++++++++++------ 1 file changed, 114 insertions(+), 24 deletions(-) diff --git a/app/db_utils/db_functions_sql.py b/app/db_utils/db_functions_sql.py index 4846d8b..86cc55f 100644 --- a/app/db_utils/db_functions_sql.py +++ b/app/db_utils/db_functions_sql.py @@ -5,18 +5,111 @@ from app.server_utils import service_helpers from app.db_utils.filtering_sql import FilteringSQL from app.db_utils.execute_sql import fetch_sql, execute_sql -from app.db_utils.data_profile import DataProfile """ Provides two classes for querying and visualizing data from a PostgreSQL database table, with support for data filtering and error annotation overlays on all chart types. +--- ColumnTypes --- +Inspects a table's schema to classify each column as numeric, categorical, or mixed-type. + --- DBOperations --- Wraps all core DB operations for a single primary table. Builds and executes multi-step CTE SQL queries that produce JSON payloads for 1D histograms, 2D histograms, and scatterplots, each annotated with per-bin/per-point error breakdowns. Also manages row-level data filters. """ +class ColumnTypes: + def __init__(self, main_table_name: str, engine): + self.numeric_cols = set() + self.categorical_mixed = set() + self.pure_categorical = set() + self.engine = engine + self.gather_numeric_cols(main_table_name) + self.gather_mixed_cols(main_table_name) + + + def gather_numeric_cols(self, main_table_name: str): + """ + Distinguishes the numeric columns from the categorical columns. + :arg: main_table_name: name of the main table. + """ + + + fetch_col_types = f'''SELECT column_name, data_type + FROM information_schema.columns + WHERE table_name = '{main_table_name}';''' + + fetched_rows = fetch_sql(fetch_col_types, False, self.engine) + if fetched_rows: + numeric_types = { + 'integer', 'bigint', 'numeric', + 'real', 'double precision', 'smallint' + } + + for row in fetched_rows: + col_name = row[0] + data_type = row[1] + + if data_type in numeric_types: + self.numeric_cols.add(col_name) + else: + self.categorical_mixed.add(col_name) + else: + raise Exception(f"No rows fetched from table: {main_table_name}") + + + def gather_mixed_cols(self, main_table_name: str): + """ + Gather the columns that are labeled as categorical but contain numeric data as well. + :arg: main_table_name: name of the main table. + """ + + # There are no categorical columns in the dataset. + if len(self.categorical_mixed) == 0: + return + + numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" + + # Initialized in the other constructor func gather_numeric_cols. This starts as all categorical columns. + # Stop early if a mixed type is found, since that makes the entire column of mixed type. + queries = [ + f"""( + SELECT '{col}' AS column_name + FROM "{main_table_name}" + WHERE "{col}" ~ {numeric_regex} + LIMIT 1 + )""" + for col in self.categorical_mixed + ] + + fetch_mixed_types = "\nUNION ALL\n".join(queries) + mixed_cols = fetch_sql(fetch_mixed_types, False, self.engine) + + mixed_col_names = set(row[0] for row in mixed_cols) if mixed_cols else set() + self.pure_categorical = self.categorical_mixed - mixed_col_names + self.categorical_mixed = mixed_col_names + + def is_categorical_col(self, col_name: str): + return col_name in self.pure_categorical + + def is_numeric_col(self, col_name: str): + """ + Determines whether the given column from the table used to construct this class is numeric. + :arg: col_name: name of the column (assumes it is from the same table used to construct this class). + :return: whether the given col_name is numeric. + """ + return col_name in self.numeric_cols + + + def is_mixed_col(self, col_name: str): + """ + Determines whether the given column from the table used to construct this class is of mixed type. + :arg: col_name: name of the column (assumes it is from the same table used to construct this class). + :return: whether the given col_name is of mixed type. + """ + return col_name in self.categorical_mixed + # Wraps up all Core DBOperations into one class using a primary main_table. class DBOperations: @@ -30,8 +123,7 @@ def __init__(self, engine): self.engine = engine self.main_table_name = None self.error_table_name = None - self.data_profile_table_name = None - self.data_profile = None + self.col_types = None self.filtering_table = None self.active_hists = {} @@ -42,12 +134,11 @@ def reset(self): """ self.main_table_name = None self.error_table_name = None - self.data_profile_table_name = None - self.data_profile = None + self.col_types = None self.filtering_table = None self.active_hists = {} - def load_table(self, main_table_name: str, error_table_name: str = None, data_profile_table_name: str = None): + def load_table(self, main_table_name: str, error_table_name: str = None): """ Loads in the main and error tables, inits the ColumnTypes and FilteringSQL objects with the new table @@ -56,8 +147,7 @@ def load_table(self, main_table_name: str, error_table_name: str = None, data_pr """ self.main_table_name = main_table_name self.error_table_name = error_table_name if error_table_name is not None else "errors_" + main_table_name - self.data_profile_table_name = data_profile_table_name if data_profile_table_name is not None else "dp_" + main_table_name - self.data_profile = DataProfile(main_table_name, self.engine) + self.col_types = ColumnTypes(main_table_name, self.engine) self.filtering_table = FilteringSQL(main_table_name, self.engine) self.active_hists = {} @@ -209,7 +299,7 @@ def gather_bins_1d_hist(self, axis_column: str, bin_count: int) -> str: :return: the query for the binning. """ - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" bin_logic = f'''CASE WHEN d.value::text ~ {numeric_regex} THEN @@ -297,7 +387,7 @@ def build_numeric_scale_data_1d_hist(self, axis_column: str, bin_count: int) -> :return: the query for the numeric scaling data. """ - if not self.data_profile.is_numeric_col(axis_column): + if not self.col_types.is_numeric_col(axis_column): return "" else: return f''', range_data AS ( @@ -332,7 +422,7 @@ def construct_1d_hist_json(self, axis_column: str) -> str: binned_data = '''SELECT (SELECT json_agg(json_build_array("ID", bin)) FROM binned_data),''' - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): return f'''{binned_data} (SELECT json_build_object( 'histograms', -- For numeric: handle mixed bins (numeric and "null") - keep bins as text @@ -430,7 +520,7 @@ def generate_2d_hist_bounds(self, bound_table_name: str, axis_column: str, col_a :return: the query to generate the bound tables. """ - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" return f''', {bound_table_name} AS ( SELECT @@ -464,7 +554,7 @@ def gather_bins_2d_hist(self, x_axis_column: str, y_axis_column: str, x_bin_coun for axis_column, bin_count, axis_alias, bounding_table in axis_info: numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): bin_logic = f'''CASE WHEN d.{axis_alias}::text ~ {numeric_regex} THEN -- Clamp bin number to 0..(bin_count-1) range @@ -564,7 +654,7 @@ def build_numeric_scale_data_2d_hist(self, bound_table_name: str, axis_column: s :return: the query for the numeric scaling data. """ - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): return f''', {scale_table_name}_range_data AS ( SELECT min_val, @@ -597,7 +687,7 @@ def construct_2d_hist_json(self, x_axis_column: str, y_axis_column: str, x_scale empty_set = r"'{}'" # Handles mixed types in x-axis. - if self.data_profile.is_numeric_col(x_axis_column): + if self.col_types.is_numeric_col(x_axis_column): json_x_type = f'''CASE WHEN x_bin ~ {numeric_regex} THEN 'numeric' ELSE 'categorical' END''' json_order_by_x = f'''CASE WHEN x_bin ~ {numeric_regex} THEN lpad(x_bin, 10, '0') ELSE x_bin END''' else: @@ -605,7 +695,7 @@ def construct_2d_hist_json(self, x_axis_column: str, y_axis_column: str, x_scale json_order_by_x = "x_bin" # Handles mixed types in y-axis. - if self.data_profile.is_numeric_col(y_axis_column): + if self.col_types.is_numeric_col(y_axis_column): json_y_type = f'''CASE WHEN y_bin ~ {numeric_regex} THEN 'numeric' ELSE 'categorical' END''' json_order_by_y = f'''CASE WHEN y_bin ~ {numeric_regex} THEN lpad(y_bin, 10, '0') ELSE y_bin END''' else: @@ -634,7 +724,7 @@ def construct_2d_hist_json(self, x_axis_column: str, y_axis_column: str, x_scale for i in range(len(json_scale_data)): scale_label, axis_column, scale_table_name, axis_bin = json_scale_data[i] - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): axis_numeric_info = f'''(SELECT COALESCE(json_agg(json_build_object('x0', x0, 'x1', x1) ORDER BY bin_num), '[]'::json) FROM {scale_table_name})''' else: @@ -781,7 +871,7 @@ def collect_scatter_axis_bounds(self, bound_table_name: str, axis_column: str, c :return: the query for aggregating scatterplot error data w/ sampled points. """ - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): return f''', {bound_table_name} AS ( SELECT @@ -810,9 +900,9 @@ def construct_scatter_json(self, x_axis_column: str, y_axis_column: str, x_col_a # Helper function to determine axis type def determine_axis_type(axis_column: str, col_alias: str) -> str: - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): return "ELSE 'numeric'" - elif self.data_profile.is_mixed_col(axis_column): + elif self.col_types.is_mixed_col(axis_column): return f"WHEN ({col_alias}::text ~ {numeric_regex}) THEN 'numeric' ELSE 'categorical'" else: return "ELSE 'categorical'" @@ -820,9 +910,9 @@ def determine_axis_type(axis_column: str, col_alias: str) -> str: # Helper function to determine JSON axis type def determine_json_axis_type(axis_column: str, col_alias: str) -> str: - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): return f"ELSE to_json({col_alias}::numeric)" - elif self.data_profile.is_mixed_col(axis_column): + elif self.col_types.is_mixed_col(axis_column): return f"WHEN ({col_alias}::text ~ {numeric_regex}) THEN to_json({col_alias}::numeric) ELSE to_json({col_alias}::text)" else: return f"ELSE to_json({col_alias}::text)" @@ -868,12 +958,12 @@ def determine_json_axis_type(axis_column: str, col_alias: str) -> str: for i in range(len(json_scale_data)): scale_label, axis_column, bounding_table, axis_alias = json_scale_data[i] - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): axis_numeric_info = f'''json_build_array( (SELECT min_val FROM {bounding_table}), (SELECT max_val + 1 FROM {bounding_table}) )''' - elif self.data_profile.is_mixed_col(axis_column): + elif self.col_types.is_mixed_col(axis_column): axis_numeric_info = f'''json_build_array( (SELECT COALESCE(MIN({axis_alias}::numeric), 0) FROM sampled_data WHERE {axis_alias}::text ~ {numeric_regex}), From d2862af6ed5cf4b826ccb07898b9944e97337132 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 19 Jul 2026 13:32:48 -0600 Subject: [PATCH 50/81] Reapply "Removed ColumnTypes from db_functions_sql.py and added data profile instance into DBOperations to get column type functions" This reverts commit 2ec0c8badfd2890d8b07d65f7e3321e553298886. --- app/db_utils/db_functions_sql.py | 138 ++++++------------------------- 1 file changed, 24 insertions(+), 114 deletions(-) diff --git a/app/db_utils/db_functions_sql.py b/app/db_utils/db_functions_sql.py index 86cc55f..4846d8b 100644 --- a/app/db_utils/db_functions_sql.py +++ b/app/db_utils/db_functions_sql.py @@ -5,111 +5,18 @@ from app.server_utils import service_helpers from app.db_utils.filtering_sql import FilteringSQL from app.db_utils.execute_sql import fetch_sql, execute_sql +from app.db_utils.data_profile import DataProfile """ Provides two classes for querying and visualizing data from a PostgreSQL database table, with support for data filtering and error annotation overlays on all chart types. ---- ColumnTypes --- -Inspects a table's schema to classify each column as numeric, categorical, or mixed-type. - --- DBOperations --- Wraps all core DB operations for a single primary table. Builds and executes multi-step CTE SQL queries that produce JSON payloads for 1D histograms, 2D histograms, and scatterplots, each annotated with per-bin/per-point error breakdowns. Also manages row-level data filters. """ -class ColumnTypes: - def __init__(self, main_table_name: str, engine): - self.numeric_cols = set() - self.categorical_mixed = set() - self.pure_categorical = set() - self.engine = engine - self.gather_numeric_cols(main_table_name) - self.gather_mixed_cols(main_table_name) - - - def gather_numeric_cols(self, main_table_name: str): - """ - Distinguishes the numeric columns from the categorical columns. - :arg: main_table_name: name of the main table. - """ - - - fetch_col_types = f'''SELECT column_name, data_type - FROM information_schema.columns - WHERE table_name = '{main_table_name}';''' - - fetched_rows = fetch_sql(fetch_col_types, False, self.engine) - if fetched_rows: - numeric_types = { - 'integer', 'bigint', 'numeric', - 'real', 'double precision', 'smallint' - } - - for row in fetched_rows: - col_name = row[0] - data_type = row[1] - - if data_type in numeric_types: - self.numeric_cols.add(col_name) - else: - self.categorical_mixed.add(col_name) - else: - raise Exception(f"No rows fetched from table: {main_table_name}") - - - def gather_mixed_cols(self, main_table_name: str): - """ - Gather the columns that are labeled as categorical but contain numeric data as well. - :arg: main_table_name: name of the main table. - """ - - # There are no categorical columns in the dataset. - if len(self.categorical_mixed) == 0: - return - - numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" - - # Initialized in the other constructor func gather_numeric_cols. This starts as all categorical columns. - # Stop early if a mixed type is found, since that makes the entire column of mixed type. - queries = [ - f"""( - SELECT '{col}' AS column_name - FROM "{main_table_name}" - WHERE "{col}" ~ {numeric_regex} - LIMIT 1 - )""" - for col in self.categorical_mixed - ] - - fetch_mixed_types = "\nUNION ALL\n".join(queries) - mixed_cols = fetch_sql(fetch_mixed_types, False, self.engine) - - mixed_col_names = set(row[0] for row in mixed_cols) if mixed_cols else set() - self.pure_categorical = self.categorical_mixed - mixed_col_names - self.categorical_mixed = mixed_col_names - - def is_categorical_col(self, col_name: str): - return col_name in self.pure_categorical - - def is_numeric_col(self, col_name: str): - """ - Determines whether the given column from the table used to construct this class is numeric. - :arg: col_name: name of the column (assumes it is from the same table used to construct this class). - :return: whether the given col_name is numeric. - """ - return col_name in self.numeric_cols - - - def is_mixed_col(self, col_name: str): - """ - Determines whether the given column from the table used to construct this class is of mixed type. - :arg: col_name: name of the column (assumes it is from the same table used to construct this class). - :return: whether the given col_name is of mixed type. - """ - return col_name in self.categorical_mixed - # Wraps up all Core DBOperations into one class using a primary main_table. class DBOperations: @@ -123,7 +30,8 @@ def __init__(self, engine): self.engine = engine self.main_table_name = None self.error_table_name = None - self.col_types = None + self.data_profile_table_name = None + self.data_profile = None self.filtering_table = None self.active_hists = {} @@ -134,11 +42,12 @@ def reset(self): """ self.main_table_name = None self.error_table_name = None - self.col_types = None + self.data_profile_table_name = None + self.data_profile = None self.filtering_table = None self.active_hists = {} - def load_table(self, main_table_name: str, error_table_name: str = None): + def load_table(self, main_table_name: str, error_table_name: str = None, data_profile_table_name: str = None): """ Loads in the main and error tables, inits the ColumnTypes and FilteringSQL objects with the new table @@ -147,7 +56,8 @@ def load_table(self, main_table_name: str, error_table_name: str = None): """ self.main_table_name = main_table_name self.error_table_name = error_table_name if error_table_name is not None else "errors_" + main_table_name - self.col_types = ColumnTypes(main_table_name, self.engine) + self.data_profile_table_name = data_profile_table_name if data_profile_table_name is not None else "dp_" + main_table_name + self.data_profile = DataProfile(main_table_name, self.engine) self.filtering_table = FilteringSQL(main_table_name, self.engine) self.active_hists = {} @@ -299,7 +209,7 @@ def gather_bins_1d_hist(self, axis_column: str, bin_count: int) -> str: :return: the query for the binning. """ - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" bin_logic = f'''CASE WHEN d.value::text ~ {numeric_regex} THEN @@ -387,7 +297,7 @@ def build_numeric_scale_data_1d_hist(self, axis_column: str, bin_count: int) -> :return: the query for the numeric scaling data. """ - if not self.col_types.is_numeric_col(axis_column): + if not self.data_profile.is_numeric_col(axis_column): return "" else: return f''', range_data AS ( @@ -422,7 +332,7 @@ def construct_1d_hist_json(self, axis_column: str) -> str: binned_data = '''SELECT (SELECT json_agg(json_build_array("ID", bin)) FROM binned_data),''' - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): return f'''{binned_data} (SELECT json_build_object( 'histograms', -- For numeric: handle mixed bins (numeric and "null") - keep bins as text @@ -520,7 +430,7 @@ def generate_2d_hist_bounds(self, bound_table_name: str, axis_column: str, col_a :return: the query to generate the bound tables. """ - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" return f''', {bound_table_name} AS ( SELECT @@ -554,7 +464,7 @@ def gather_bins_2d_hist(self, x_axis_column: str, y_axis_column: str, x_bin_coun for axis_column, bin_count, axis_alias, bounding_table in axis_info: numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): bin_logic = f'''CASE WHEN d.{axis_alias}::text ~ {numeric_regex} THEN -- Clamp bin number to 0..(bin_count-1) range @@ -654,7 +564,7 @@ def build_numeric_scale_data_2d_hist(self, bound_table_name: str, axis_column: s :return: the query for the numeric scaling data. """ - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): return f''', {scale_table_name}_range_data AS ( SELECT min_val, @@ -687,7 +597,7 @@ def construct_2d_hist_json(self, x_axis_column: str, y_axis_column: str, x_scale empty_set = r"'{}'" # Handles mixed types in x-axis. - if self.col_types.is_numeric_col(x_axis_column): + if self.data_profile.is_numeric_col(x_axis_column): json_x_type = f'''CASE WHEN x_bin ~ {numeric_regex} THEN 'numeric' ELSE 'categorical' END''' json_order_by_x = f'''CASE WHEN x_bin ~ {numeric_regex} THEN lpad(x_bin, 10, '0') ELSE x_bin END''' else: @@ -695,7 +605,7 @@ def construct_2d_hist_json(self, x_axis_column: str, y_axis_column: str, x_scale json_order_by_x = "x_bin" # Handles mixed types in y-axis. - if self.col_types.is_numeric_col(y_axis_column): + if self.data_profile.is_numeric_col(y_axis_column): json_y_type = f'''CASE WHEN y_bin ~ {numeric_regex} THEN 'numeric' ELSE 'categorical' END''' json_order_by_y = f'''CASE WHEN y_bin ~ {numeric_regex} THEN lpad(y_bin, 10, '0') ELSE y_bin END''' else: @@ -724,7 +634,7 @@ def construct_2d_hist_json(self, x_axis_column: str, y_axis_column: str, x_scale for i in range(len(json_scale_data)): scale_label, axis_column, scale_table_name, axis_bin = json_scale_data[i] - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): axis_numeric_info = f'''(SELECT COALESCE(json_agg(json_build_object('x0', x0, 'x1', x1) ORDER BY bin_num), '[]'::json) FROM {scale_table_name})''' else: @@ -871,7 +781,7 @@ def collect_scatter_axis_bounds(self, bound_table_name: str, axis_column: str, c :return: the query for aggregating scatterplot error data w/ sampled points. """ - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): return f''', {bound_table_name} AS ( SELECT @@ -900,9 +810,9 @@ def construct_scatter_json(self, x_axis_column: str, y_axis_column: str, x_col_a # Helper function to determine axis type def determine_axis_type(axis_column: str, col_alias: str) -> str: - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): return "ELSE 'numeric'" - elif self.col_types.is_mixed_col(axis_column): + elif self.data_profile.is_mixed_col(axis_column): return f"WHEN ({col_alias}::text ~ {numeric_regex}) THEN 'numeric' ELSE 'categorical'" else: return "ELSE 'categorical'" @@ -910,9 +820,9 @@ def determine_axis_type(axis_column: str, col_alias: str) -> str: # Helper function to determine JSON axis type def determine_json_axis_type(axis_column: str, col_alias: str) -> str: - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): return f"ELSE to_json({col_alias}::numeric)" - elif self.col_types.is_mixed_col(axis_column): + elif self.data_profile.is_mixed_col(axis_column): return f"WHEN ({col_alias}::text ~ {numeric_regex}) THEN to_json({col_alias}::numeric) ELSE to_json({col_alias}::text)" else: return f"ELSE to_json({col_alias}::text)" @@ -958,12 +868,12 @@ def determine_json_axis_type(axis_column: str, col_alias: str) -> str: for i in range(len(json_scale_data)): scale_label, axis_column, bounding_table, axis_alias = json_scale_data[i] - if self.col_types.is_numeric_col(axis_column): + if self.data_profile.is_numeric_col(axis_column): axis_numeric_info = f'''json_build_array( (SELECT min_val FROM {bounding_table}), (SELECT max_val + 1 FROM {bounding_table}) )''' - elif self.col_types.is_mixed_col(axis_column): + elif self.data_profile.is_mixed_col(axis_column): axis_numeric_info = f'''json_build_array( (SELECT COALESCE(MIN({axis_alias}::numeric), 0) FROM sampled_data WHERE {axis_alias}::text ~ {numeric_regex}), From af51b4dc917c6b2ad9caf6efa0605618b6ce9ea1 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 19 Jul 2026 15:28:23 -0600 Subject: [PATCH 51/81] Revert "Added functions to get list different column types" This reverts commit b14815a91d54b50d45faf4632d52028607f21d62. --- app/db_utils/data_profile.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index f2de3d4..2300984 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -529,13 +529,5 @@ def get_column_names(self): return all_cols - def get_numeric_cols(self): - return list(self.col_types.numeric_cols) - - def get_categorical_cols(self): - return list(self.col_types.categorical_cols) - - def get_mixed_cols(self): - return list(self.col_types.mixed_cols) From d91e8f738a275b83f96d7990f1e61f80ca514f05 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 19 Jul 2026 15:31:47 -0600 Subject: [PATCH 52/81] added pure numeric and pure categorical columns to column_types.py --- app/db_utils/column_types.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/db_utils/column_types.py b/app/db_utils/column_types.py index 855b540..7970083 100644 --- a/app/db_utils/column_types.py +++ b/app/db_utils/column_types.py @@ -21,6 +21,9 @@ def __init__(self, main_table_name: str, engine): self.gather_mixed_cols(main_table_name) self.categorize_mixed_cols(main_table_name) + self.pure_numeric_columns = self.numeric_cols.difference(self.mixed_cols) + self.pure_categorical_columns = self.categorical_cols.difference(self.mixed_cols) + def get_col_type(self, column_name): """ :param column_name: Name of the column for which the type is being checked From bbc3411e5045a4073bcd0a8659b07ae5be2aa29b Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 19 Jul 2026 15:39:41 -0600 Subject: [PATCH 53/81] Revert "Moved ColumnTypes class into data_profile.py and added column types functions into the DataProfile class" This reverts commit 9f8dee71cd8111f676d18f79e1aee16f71c3b584. --- app/ai_utils/ai_data_attributes.json | 1 + app/db_utils/data_profile.py | 181 +++-------------------- provided_datasets/create_mari_dataset.py | 15 ++ provided_datasets/mari_dataset.csv | 7 + 4 files changed, 42 insertions(+), 162 deletions(-) create mode 100644 app/ai_utils/ai_data_attributes.json create mode 100644 provided_datasets/create_mari_dataset.py create mode 100644 provided_datasets/mari_dataset.csv diff --git a/app/ai_utils/ai_data_attributes.json b/app/ai_utils/ai_data_attributes.json new file mode 100644 index 0000000..ed27ea4 --- /dev/null +++ b/app/ai_utils/ai_data_attributes.json @@ -0,0 +1 @@ +{"index": {"data_type": "int64", "num_na": 0, "count": 38090, "mean": 19044.5, "median": 19044.5, "min": 0, "max": 38089, "var": 120907182.5, "iqr": 19044.5, "std": 10995.780213336388, "skew": 1.5799198991778757e-16, "median_absolute_deviation": 9522, "tukeys_fence": [-19044.5, 57133.5]}, "ID": {"data_type": "int64", "num_na": 0, "count": 38090, "mean": 23610.64715148333, "median": 23522.5, "min": 0, "max": 47701, "var": 186508862.89536914, "iqr": 23510.5, "std": 13656.824773547076, "skew": 0.013420674993980477, "median_absolute_deviation": 11756, "tukeys_fence": [-23438.5, 70603.5]}, "Hobby": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 2, "category_count": {"Yes": 31125, "No": 6965}}, "Country": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 20, "category_count": {"United States": 12941, "India": 4091, "United Kingdom": 3794, "Germany": 3362, "Canada": 1983, "France": 1447, "Russian Federation": 1209, "Australia": 1180, "Brazil": 1151, "Netherlands": 1001, "Spain": 980, "Poland": 935, "Italy": 748, "Sweden": 678, "Switzerland": 553, "Israel": 442, "Ukraine": 438, "Austria": 393, "Belgium": 383, "Turkey": 381}}, "Student": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 3, "category_count": {"No": 31785, "Yes, full-time": 4174, "Yes, part-time": 2131}}, "FormalEducation": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 11, "category_count": {"Bachelor's degree (BA, BS, B.Eng., etc.)": 18463, "Master's degree (MA, MS, M.Eng., MBA, etc.)": 9174, "Some college/university study without earning a degree": 4824, "Secondary school (e.g. American high school, German Realschule or Gymnasium, etc.)": 2407, "Associate degree": 1278, "Other doctoral degree (Ph.D, Ed.D., etc.)": 1050, "Professional degree (JD, MD, etc.)": 443, "Primary/elementary school": 305, "I never completed any formal education": 134, "'00'": 7, "'4'": 5}}, "UndergradMajor": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 12, "category_count": {"Computer science, computer engineering, or software engineering": 23839, "Another engineering discipline (ex. civil, electrical, mechanical)": 3215, "Information systems, information technology, or system administration": 2873, "A natural science (ex. biology, chemistry, physics)": 1761, "Mathematics or statistics": 1531, "A humanities discipline (ex. literature, history, philosophy)": 993, "Web development or web design": 963, "A business discipline (ex. accounting, finance, marketing)": 916, "A social science (ex. anthropology, psychology, political science)": 843, "Fine arts or performing arts (ex. graphic design, music, studio art)": 703, "I never declared a major": 340, "A health science (ex. nursing, pharmacy, radiology)": 113}}, "DevType": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 20, "category_count": {"Back-end developer": 22987, "Full-stack developer": 3329, "Front-end developer": 2307, "Mobile developer": 1451, "Desktop or enterprise applications developer": 1355, "Data or business analyst": 1149, "Data scientist or machine learning specialist": 1001, "Designer": 857, "DevOps specialist": 667, "Embedded applications or devices developer": 600, "Database administrator": 496, "Student": 409, "Engineering manager": 372, "C-suite executive (CEO, CTO, etc.)": 372, "QA or test developer": 264, "Educator or academic researcher": 187, "Game or graphics developer": 149, "System administrator": 105, "Product manager": 19, "Marketing or sales professional": 14}}, "YearsCoding": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 11, "category_count": {"6-8 years": 8064, "3-5 years": 7516, "9-11 years": 5544, "12-14 years": 4050, "15-17 years": 3189, "18-20 years": 2669, "0-2 years": 2348, "30 or more years": 1799, "21-23 years": 1395, "24-26 years": 980, "27-29 years": 536}}, "HoursComputer": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 6, "category_count": {"9 - 12 hours": 21280, "5 - 8 hours": 11483, "Over 12 hours": 4731, "1 - 4 hours": 542, "Less than 1 hour": 42, "UNKNOWN": 12}}, "Exercise": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 5, "category_count": {"I don't typically exercise": 13908, "1 - 2 times per week": 11065, "3 - 4 times per week": 7925, "Daily or almost every day": 5185, "UNKNOWN": 7}}, "Gender": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 16, "category_count": {"Male": 35200, "Female": 2293, "Non-binary, genderqueer, or gender non-conforming": 174, "Female;Transgender": 96, "Male;Non-binary, genderqueer, or gender non-conforming": 83, "Transgender": 54, "Transgender;Non-binary, genderqueer, or gender non-conforming": 44, "Female;Non-binary, genderqueer, or gender non-conforming": 39, "Female;Male": 24, "Male;Transgender": 23, "UNKNOWN": 22, "Female;Transgender;Non-binary, genderqueer, or gender non-conforming": 16, "Female;Male;Transgender;Non-binary, genderqueer, or gender non-conforming": 13, "Female;Male;Transgender": 5, "Male;Transgender;Non-binary, genderqueer, or gender non-conforming": 3, "Female;Male;Non-binary, genderqueer, or gender non-conforming": 1}}, "SexualOrientation": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 4, "category_count": {"Straight or heterosexual": 35377, "Bisexual or Queer": 1407, "Gay or Lesbian": 931, "Asexual": 375}}, "EducationParents": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 10, "category_count": {"Bachelor's degree (BA, BS, B.Eng., etc.)": 11181, "Master's degree (MA, MS, M.Eng., MBA, etc.)": 8557, "Secondary school (e.g. American high school, German Realschule or Gymnasium, etc.)": 6557, "Some college/university study without earning a degree": 3497, "Other doctoral degree (Ph.D, Ed.D., etc.)": 2396, "Associate degree": 1868, "Primary/elementary school": 1750, "Professional degree (JD, MD, etc.)": 1693, "They never completed any formal education": 560, "UNKNOWN": 31}}, "RaceEthnicity": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 7, "category_count": {"White or of European descent": 29558, "South Asian": 3471, "Hispanic or Latino/Latina": 1855, "East Asian": 1311, "Middle Eastern": 1011, "Black or of African descent": 673, "Native American, Pacific Islander, or Indigenous Australian": 211}}, "Dependents": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 3, "category_count": {"No": 26556, "Yes": 11509, "UNKNOWN": 25}}, "Continent": {"data_type": "str", "num_na": 9, "count": 38081, "num_unique": 6, "category_count": {"EU": 15918, "North America": 14919, "AS": 4913, "OC": 1180, "SA": 1151, "N/A": 9}}, "Age": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 11, "category_count": {"25 - 34 years old": 19848, "18 - 24 years old": 7636, "35 - 44 years old": 7495, "45 - 54 years old": 2101, "55 - 64 years old": 574, "Under 18 years old": 353, "65 years or older": 62, "UNKNOWN": 18, "0 years old": 1, "21.5": 1, "0": 1}}, "ConvertedSalary": {"data_type": "int64", "num_na": 0, "count": 38090, "mean": 105310.73105802048, "median": 62507.0, "min": 0, "max": 2000000, "var": 45762158379.79209, "iqr": 68212.0, "std": 213920.9161811722, "skew": 6.135605238069827, "median_absolute_deviation": 34119, "tukeys_fence": [-70530.0, 202318.0]}, "HDI": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 3, "category_count": {"High": 29885, "Medium": 6616, "Low": 1589}}, "GDP": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 3, "category_count": {"Medium": 16944, "High": 12941, "Low": 8205}}, "GINI": {"data_type": "str", "num_na": 0, "count": 38090, "num_unique": 3, "category_count": {"High": 20653, "Medium": 14429, "Low": 3008}}} \ No newline at end of file diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index 2300984..add561c 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -22,127 +22,6 @@ def to_scalar(val): return val.item() return val -""" ---- ColumnTypes --- -Inspects a table's schema to classify each column as numeric, categorical, or mixed-type. -""" - -class ColumnTypes: - def __init__(self, main_table_name: str, engine): - # Cols where majority of the rows are numeric - self.numeric_cols = set() - self.mixed_cols = set() - # Cols where majority of the rows are categorical - self.categorical_cols = set() - self.engine = engine - - self.numeric_types = [ - 'integer', 'bigint', 'numeric', - 'real', 'double precision', 'smallint' - ] - self.gather_numeric_cols(main_table_name) - self.gather_mixed_cols(main_table_name) - self.categorize_mixed_cols(main_table_name) - - def gather_numeric_cols(self, main_table_name: str): - """ - Distinguishes the numeric columns from the categorical columns. - :arg: main_table_name: name of the main table. - """ - - fetch_col_types = f'''SELECT column_name, data_type - FROM information_schema.columns - WHERE table_name = '{main_table_name}';''' - - fetched_rows = fetch_sql(fetch_col_types, False, self.engine) - if fetched_rows: - - for row in fetched_rows: - col_name = row[0] - # This datatype will only be numeric if the whole column is numeric - data_type = row[1] - - if data_type in self.numeric_types: - self.numeric_cols.add(col_name) - else: - self.mixed_cols.add(col_name) - else: - raise Exception(f"No rows fetched from table: {main_table_name}") - - - def gather_mixed_cols(self, main_table_name: str): - """ - Gather the columns that are labeled as categorical but contain numeric data as well. - :arg: main_table_name: name of the main table. - """ - - # There are no categorical columns in the dataset. - if len(self.mixed_cols) == 0: - return - - numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" - - # Initialized in the other constructor func gather_numeric_cols. This starts as all categorical columns. - # Stop early if a mixed type is found, since that makes the entire column of mixed type. - queries = [ - f"""( - SELECT '{col}' AS column_name - FROM "{main_table_name}" - WHERE pg_input_is_valid("{col}", 'numeric') - LIMIT 1 - )""" - for col in self.mixed_cols - ] - - fetch_mixed_types = "\nUNION ALL\n".join(queries) - mixed_cols = fetch_sql(fetch_mixed_types, False, self.engine) - - mixed_col_names = set(row[0] for row in mixed_cols) if mixed_cols else set() - self.categorical_cols = self.mixed_cols - mixed_col_names - self.mixed_cols = mixed_col_names - - def categorize_mixed_cols(self, main_table_name: str): - """ - Categorizes the mixed columns into numeric and categorical based on the majority of their values. - :arg: main_table_name: name of the main table. - """ - for col in self.mixed_cols: - query = f""" - SELECT - SUM(CASE WHEN pg_input_is_valid("{col}", \'numeric\' )THEN 1 ELSE 0 END) AS numeric_count, - COUNT(*) AS total_count - FROM "{main_table_name}"; - """ - result = fetch_sql(query, False, self.engine) - if result: - numeric_count, total_count = result[0] - if numeric_count > total_count / 2: - self.numeric_cols.add(col) - else: - self.categorical_cols.add(col) - else: - raise Exception(f"No rows fetched for column: {col} in table: {main_table_name}") - - def is_categorical_col(self, col_name: str): - return col_name in self.categorical_cols - - def is_numeric_col(self, col_name: str): - """ - Determines whether the given column from the table used to construct this class is numeric. - :arg: col_name: name of the column (assumes it is from the same table used to construct this class). - :return: whether the given col_name is numeric. - """ - return col_name in self.numeric_cols - - - def is_mixed_col(self, col_name: str): - """ - Determines whether the given column from the table used to construct this class is of mixed type. - :arg: col_name: name of the column (assumes it is from the same table used to construct this class). - :return: whether the given col_name is of mixed type. - """ - return col_name in self.mixed_cols - class DataProfile: """ Class that handles queries to get summary stats about the main data table. @@ -483,51 +362,29 @@ def _calculate_category_count_dict(self, column_name): return category_counts - def get_col_type(self, column_name): - """ - :param column_name: Name of the column for which the type is being checked - :return: The type of the column in a string - """ - if self.col_types.is_numeric_col(column_name): - return "numeric" - elif self.col_types.is_categorical_col(column_name): - return "categorical" - elif self.col_types.is_mixed_col(column_name): - return "mixed" - else: - return None - - def is_numeric_col(self, column_name): - """ - :param column_name: Name of the column for which the type is being checked - :return: True if the column is numeric, False otherwise - """ - return self.col_types.is_numeric_col(column_name) - - def is_categorical_col(self, column_name): - """ - :param column_name: Name of the column for which the type is being checked - :return: True if the column is categorical, False otherwise - """ - return self.col_types.is_categorical_col(column_name) - - def is_mixed_col(self, column_name): + # Dict mapping from class to error types to error counts + # TODO: Implement SQL query version + def _calculate_class_error_count_dict(self, column_name): """ - :param column_name: Name of the column for which the type is being checked - :return: True if the column is mixed, False otherwise + :param column_name: Name of the column for which the class error count is being calculated + :return: The class error count dict ({"Male": {"missing": 10, "mismatch": 5, ...}, "Female": {"missing": 10, "mismatch": 5, ...}}) """ - return self.col_types.is_mixed_col(column_name) + # TODO: Implement SQL query version + print("Calculating class error counts manually using data...") + self.load_error_df() - def get_column_names(self): - """ - :return: List of column names - """ - # uses the sets of column names from the ColumnTypes class to get all column names - all_cols = [list(self.col_types.numeric_cols), list(self.col_types.categorical_cols), - list(self.col_types.mixed_cols)] - - return all_cols + counts_by_column = {} + if not self._error_df.empty: # If error_df is empty (no errors in data selection) + counts_by_column = ( + self._error_df.groupby(['column_id', 'error_type']) + .size() + .unstack(fill_value=0) + .to_dict(orient='index') + ) + if counts_by_column is not None: + counts_by_column = json.dumps(counts_by_column) + return counts_by_column diff --git a/provided_datasets/create_mari_dataset.py b/provided_datasets/create_mari_dataset.py new file mode 100644 index 0000000..fd698ed --- /dev/null +++ b/provided_datasets/create_mari_dataset.py @@ -0,0 +1,15 @@ +import csv + +data = [ + {"id": 1, "name": "Alex Smith", "age": 29, "city": "Salt Lake City"}, + {"id": 2, "name": "Jordan Lee", "age": 34, "city": "New York"}, + {"id": 3, "name": "Taylor Kim", "age": "one", "city": "Chicago"}, + {"id": 4, "name": "Lenny Kim", "age": 29, "city": "Portland"}, + {"id": 5, "name": "John Doe", "age": 32, "city": "Vancouver"}, + {"id": 6, "name": "Mari Martinez", "age": 20, "city": "Chicago"}, + ] + +with open("mari_dataset.csv", "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=data[0].keys()) + writer.writeheader() + writer.writerows(data) diff --git a/provided_datasets/mari_dataset.csv b/provided_datasets/mari_dataset.csv new file mode 100644 index 0000000..e1b3ac2 --- /dev/null +++ b/provided_datasets/mari_dataset.csv @@ -0,0 +1,7 @@ +id,name,age,city +1,Alex Smith,29,Salt Lake City +2,Jordan Lee,34,New York +3,Taylor Kim,one,Chicago +4,Lenny Kim,29,Portland +5,John Doe,32,Vancouver +6,Mari Martinez,20,Chicago From 78c572f82df1733d3f11d807dd934f75e811f9f5 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Sun, 19 Jul 2026 15:39:58 -0600 Subject: [PATCH 54/81] Revert "Removed ColumnTypes from db_functions_sql.py and added data profile instance into DBOperations to get column type functions" This reverts commit 44c1cf7098d6694d120e5e8d1d6246f40b8fcc95. --- app/db_utils/db_functions_sql.py | 138 +++++++++++++++++++++++++------ 1 file changed, 114 insertions(+), 24 deletions(-) diff --git a/app/db_utils/db_functions_sql.py b/app/db_utils/db_functions_sql.py index 4846d8b..86cc55f 100644 --- a/app/db_utils/db_functions_sql.py +++ b/app/db_utils/db_functions_sql.py @@ -5,18 +5,111 @@ from app.server_utils import service_helpers from app.db_utils.filtering_sql import FilteringSQL from app.db_utils.execute_sql import fetch_sql, execute_sql -from app.db_utils.data_profile import DataProfile """ Provides two classes for querying and visualizing data from a PostgreSQL database table, with support for data filtering and error annotation overlays on all chart types. +--- ColumnTypes --- +Inspects a table's schema to classify each column as numeric, categorical, or mixed-type. + --- DBOperations --- Wraps all core DB operations for a single primary table. Builds and executes multi-step CTE SQL queries that produce JSON payloads for 1D histograms, 2D histograms, and scatterplots, each annotated with per-bin/per-point error breakdowns. Also manages row-level data filters. """ +class ColumnTypes: + def __init__(self, main_table_name: str, engine): + self.numeric_cols = set() + self.categorical_mixed = set() + self.pure_categorical = set() + self.engine = engine + self.gather_numeric_cols(main_table_name) + self.gather_mixed_cols(main_table_name) + + + def gather_numeric_cols(self, main_table_name: str): + """ + Distinguishes the numeric columns from the categorical columns. + :arg: main_table_name: name of the main table. + """ + + + fetch_col_types = f'''SELECT column_name, data_type + FROM information_schema.columns + WHERE table_name = '{main_table_name}';''' + + fetched_rows = fetch_sql(fetch_col_types, False, self.engine) + if fetched_rows: + numeric_types = { + 'integer', 'bigint', 'numeric', + 'real', 'double precision', 'smallint' + } + + for row in fetched_rows: + col_name = row[0] + data_type = row[1] + + if data_type in numeric_types: + self.numeric_cols.add(col_name) + else: + self.categorical_mixed.add(col_name) + else: + raise Exception(f"No rows fetched from table: {main_table_name}") + + + def gather_mixed_cols(self, main_table_name: str): + """ + Gather the columns that are labeled as categorical but contain numeric data as well. + :arg: main_table_name: name of the main table. + """ + + # There are no categorical columns in the dataset. + if len(self.categorical_mixed) == 0: + return + + numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" + + # Initialized in the other constructor func gather_numeric_cols. This starts as all categorical columns. + # Stop early if a mixed type is found, since that makes the entire column of mixed type. + queries = [ + f"""( + SELECT '{col}' AS column_name + FROM "{main_table_name}" + WHERE "{col}" ~ {numeric_regex} + LIMIT 1 + )""" + for col in self.categorical_mixed + ] + + fetch_mixed_types = "\nUNION ALL\n".join(queries) + mixed_cols = fetch_sql(fetch_mixed_types, False, self.engine) + + mixed_col_names = set(row[0] for row in mixed_cols) if mixed_cols else set() + self.pure_categorical = self.categorical_mixed - mixed_col_names + self.categorical_mixed = mixed_col_names + + def is_categorical_col(self, col_name: str): + return col_name in self.pure_categorical + + def is_numeric_col(self, col_name: str): + """ + Determines whether the given column from the table used to construct this class is numeric. + :arg: col_name: name of the column (assumes it is from the same table used to construct this class). + :return: whether the given col_name is numeric. + """ + return col_name in self.numeric_cols + + + def is_mixed_col(self, col_name: str): + """ + Determines whether the given column from the table used to construct this class is of mixed type. + :arg: col_name: name of the column (assumes it is from the same table used to construct this class). + :return: whether the given col_name is of mixed type. + """ + return col_name in self.categorical_mixed + # Wraps up all Core DBOperations into one class using a primary main_table. class DBOperations: @@ -30,8 +123,7 @@ def __init__(self, engine): self.engine = engine self.main_table_name = None self.error_table_name = None - self.data_profile_table_name = None - self.data_profile = None + self.col_types = None self.filtering_table = None self.active_hists = {} @@ -42,12 +134,11 @@ def reset(self): """ self.main_table_name = None self.error_table_name = None - self.data_profile_table_name = None - self.data_profile = None + self.col_types = None self.filtering_table = None self.active_hists = {} - def load_table(self, main_table_name: str, error_table_name: str = None, data_profile_table_name: str = None): + def load_table(self, main_table_name: str, error_table_name: str = None): """ Loads in the main and error tables, inits the ColumnTypes and FilteringSQL objects with the new table @@ -56,8 +147,7 @@ def load_table(self, main_table_name: str, error_table_name: str = None, data_pr """ self.main_table_name = main_table_name self.error_table_name = error_table_name if error_table_name is not None else "errors_" + main_table_name - self.data_profile_table_name = data_profile_table_name if data_profile_table_name is not None else "dp_" + main_table_name - self.data_profile = DataProfile(main_table_name, self.engine) + self.col_types = ColumnTypes(main_table_name, self.engine) self.filtering_table = FilteringSQL(main_table_name, self.engine) self.active_hists = {} @@ -209,7 +299,7 @@ def gather_bins_1d_hist(self, axis_column: str, bin_count: int) -> str: :return: the query for the binning. """ - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" bin_logic = f'''CASE WHEN d.value::text ~ {numeric_regex} THEN @@ -297,7 +387,7 @@ def build_numeric_scale_data_1d_hist(self, axis_column: str, bin_count: int) -> :return: the query for the numeric scaling data. """ - if not self.data_profile.is_numeric_col(axis_column): + if not self.col_types.is_numeric_col(axis_column): return "" else: return f''', range_data AS ( @@ -332,7 +422,7 @@ def construct_1d_hist_json(self, axis_column: str) -> str: binned_data = '''SELECT (SELECT json_agg(json_build_array("ID", bin)) FROM binned_data),''' - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): return f'''{binned_data} (SELECT json_build_object( 'histograms', -- For numeric: handle mixed bins (numeric and "null") - keep bins as text @@ -430,7 +520,7 @@ def generate_2d_hist_bounds(self, bound_table_name: str, axis_column: str, col_a :return: the query to generate the bound tables. """ - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" return f''', {bound_table_name} AS ( SELECT @@ -464,7 +554,7 @@ def gather_bins_2d_hist(self, x_axis_column: str, y_axis_column: str, x_bin_coun for axis_column, bin_count, axis_alias, bounding_table in axis_info: numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): bin_logic = f'''CASE WHEN d.{axis_alias}::text ~ {numeric_regex} THEN -- Clamp bin number to 0..(bin_count-1) range @@ -564,7 +654,7 @@ def build_numeric_scale_data_2d_hist(self, bound_table_name: str, axis_column: s :return: the query for the numeric scaling data. """ - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): return f''', {scale_table_name}_range_data AS ( SELECT min_val, @@ -597,7 +687,7 @@ def construct_2d_hist_json(self, x_axis_column: str, y_axis_column: str, x_scale empty_set = r"'{}'" # Handles mixed types in x-axis. - if self.data_profile.is_numeric_col(x_axis_column): + if self.col_types.is_numeric_col(x_axis_column): json_x_type = f'''CASE WHEN x_bin ~ {numeric_regex} THEN 'numeric' ELSE 'categorical' END''' json_order_by_x = f'''CASE WHEN x_bin ~ {numeric_regex} THEN lpad(x_bin, 10, '0') ELSE x_bin END''' else: @@ -605,7 +695,7 @@ def construct_2d_hist_json(self, x_axis_column: str, y_axis_column: str, x_scale json_order_by_x = "x_bin" # Handles mixed types in y-axis. - if self.data_profile.is_numeric_col(y_axis_column): + if self.col_types.is_numeric_col(y_axis_column): json_y_type = f'''CASE WHEN y_bin ~ {numeric_regex} THEN 'numeric' ELSE 'categorical' END''' json_order_by_y = f'''CASE WHEN y_bin ~ {numeric_regex} THEN lpad(y_bin, 10, '0') ELSE y_bin END''' else: @@ -634,7 +724,7 @@ def construct_2d_hist_json(self, x_axis_column: str, y_axis_column: str, x_scale for i in range(len(json_scale_data)): scale_label, axis_column, scale_table_name, axis_bin = json_scale_data[i] - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): axis_numeric_info = f'''(SELECT COALESCE(json_agg(json_build_object('x0', x0, 'x1', x1) ORDER BY bin_num), '[]'::json) FROM {scale_table_name})''' else: @@ -781,7 +871,7 @@ def collect_scatter_axis_bounds(self, bound_table_name: str, axis_column: str, c :return: the query for aggregating scatterplot error data w/ sampled points. """ - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): return f''', {bound_table_name} AS ( SELECT @@ -810,9 +900,9 @@ def construct_scatter_json(self, x_axis_column: str, y_axis_column: str, x_col_a # Helper function to determine axis type def determine_axis_type(axis_column: str, col_alias: str) -> str: - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): return "ELSE 'numeric'" - elif self.data_profile.is_mixed_col(axis_column): + elif self.col_types.is_mixed_col(axis_column): return f"WHEN ({col_alias}::text ~ {numeric_regex}) THEN 'numeric' ELSE 'categorical'" else: return "ELSE 'categorical'" @@ -820,9 +910,9 @@ def determine_axis_type(axis_column: str, col_alias: str) -> str: # Helper function to determine JSON axis type def determine_json_axis_type(axis_column: str, col_alias: str) -> str: - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): return f"ELSE to_json({col_alias}::numeric)" - elif self.data_profile.is_mixed_col(axis_column): + elif self.col_types.is_mixed_col(axis_column): return f"WHEN ({col_alias}::text ~ {numeric_regex}) THEN to_json({col_alias}::numeric) ELSE to_json({col_alias}::text)" else: return f"ELSE to_json({col_alias}::text)" @@ -868,12 +958,12 @@ def determine_json_axis_type(axis_column: str, col_alias: str) -> str: for i in range(len(json_scale_data)): scale_label, axis_column, bounding_table, axis_alias = json_scale_data[i] - if self.data_profile.is_numeric_col(axis_column): + if self.col_types.is_numeric_col(axis_column): axis_numeric_info = f'''json_build_array( (SELECT min_val FROM {bounding_table}), (SELECT max_val + 1 FROM {bounding_table}) )''' - elif self.data_profile.is_mixed_col(axis_column): + elif self.col_types.is_mixed_col(axis_column): axis_numeric_info = f'''json_build_array( (SELECT COALESCE(MIN({axis_alias}::numeric), 0) FROM sampled_data WHERE {axis_alias}::text ~ {numeric_regex}), From bfdfc384d21f3e290a830007b36016f0b0dd5256 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 20 Jul 2026 10:54:40 -0600 Subject: [PATCH 55/81] Reverted back to the old column_types function implementation. A fix is still a WIP, but focusing on implementing the ai assistant because the fix is taking too long --- app/db_utils/column_types.py | 87 ++++++++-------- app/db_utils/data_profile.py | 5 +- app/db_utils/db_functions_sql.py | 98 +------------------ app/routes/pgraph_routes.py | 2 +- app/routes/plot_routes.py | 6 +- app/routes/routes.py | 14 +-- .../data_attribute_summary_integration.py | 2 +- app/server_utils/service_helpers.py | 4 +- 8 files changed, 67 insertions(+), 151 deletions(-) diff --git a/app/db_utils/column_types.py b/app/db_utils/column_types.py index 7970083..be087e4 100644 --- a/app/db_utils/column_types.py +++ b/app/db_utils/column_types.py @@ -1,62 +1,41 @@ -from app.db_utils.execute_sql import fetch_sql -""" ---- ColumnTypes --- -Inspects a table's schema to classify each column as numeric, categorical, or mixed-type. -""" class ColumnTypes: def __init__(self, main_table_name: str, engine): - # Cols where majority of the rows are numeric self.numeric_cols = set() self.mixed_cols = set() - # Cols where majority of the rows are categorical - self.categorical_cols = set() + self.categorical_mixed_cols = set() + self.numeric_mixed_cols = set() + self.pure_categorical = set() self.engine = engine - - self.numeric_types = [ - 'integer', 'bigint', 'numeric', - 'real', 'double precision', 'smallint' - ] self.gather_numeric_cols(main_table_name) self.gather_mixed_cols(main_table_name) self.categorize_mixed_cols(main_table_name) - self.pure_numeric_columns = self.numeric_cols.difference(self.mixed_cols) - self.pure_categorical_columns = self.categorical_cols.difference(self.mixed_cols) - - def get_col_type(self, column_name): - """ - :param column_name: Name of the column for which the type is being checked - :return: The type of the column in a string - """ - if self.is_numeric_col(column_name): - return "numeric" - elif self.is_categorical_col(column_name): - return "categorical" - elif self.is_mixed_col(column_name): - return "mixed" - else: - return None def gather_numeric_cols(self, main_table_name: str): + from app.db_utils.execute_sql import fetch_sql """ Distinguishes the numeric columns from the categorical columns. :arg: main_table_name: name of the main table. """ + fetch_col_types = f'''SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '{main_table_name}';''' fetched_rows = fetch_sql(fetch_col_types, False, self.engine) if fetched_rows: + numeric_types = { + 'integer', 'bigint', 'numeric', + 'real', 'double precision', 'smallint' + } for row in fetched_rows: col_name = row[0] - # This datatype will only be numeric if the whole column is numeric data_type = row[1] - if data_type in self.numeric_types: + if data_type in numeric_types: self.numeric_cols.add(col_name) else: self.mixed_cols.add(col_name) @@ -65,6 +44,7 @@ def gather_numeric_cols(self, main_table_name: str): def gather_mixed_cols(self, main_table_name: str): + from app.db_utils.execute_sql import fetch_sql """ Gather the columns that are labeled as categorical but contain numeric data as well. :arg: main_table_name: name of the main table. @@ -80,10 +60,10 @@ def gather_mixed_cols(self, main_table_name: str): # Stop early if a mixed type is found, since that makes the entire column of mixed type. queries = [ f"""( - SELECT '{col}' AS column_name - FROM "{main_table_name}" - WHERE pg_input_is_valid("{col}", 'numeric') - LIMIT 1 + SELECT '{col}' AS column_name + FROM "{main_table_name}" + WHERE "{col}" ~ {numeric_regex} + LIMIT 1 )""" for col in self.mixed_cols ] @@ -92,7 +72,7 @@ def gather_mixed_cols(self, main_table_name: str): mixed_cols = fetch_sql(fetch_mixed_types, False, self.engine) mixed_col_names = set(row[0] for row in mixed_cols) if mixed_cols else set() - self.categorical_cols = self.mixed_cols - mixed_col_names + self.pure_categorical = self.mixed_cols - mixed_col_names self.mixed_cols = mixed_col_names def categorize_mixed_cols(self, main_table_name: str): @@ -100,25 +80,27 @@ def categorize_mixed_cols(self, main_table_name: str): Categorizes the mixed columns into numeric and categorical based on the majority of their values. :arg: main_table_name: name of the main table. """ + from app.db_utils.execute_sql import fetch_sql + for col in self.mixed_cols: query = f""" - SELECT - SUM(CASE WHEN pg_input_is_valid("{col}", \'numeric\' )THEN 1 ELSE 0 END) AS numeric_count, - COUNT(*) AS total_count - FROM "{main_table_name}"; - """ + SELECT + SUM(CASE WHEN pg_input_is_valid("{col}", \'numeric\' )THEN 1 ELSE 0 END) AS numeric_count, + COUNT(*) AS total_count + FROM "{main_table_name}"; + """ result = fetch_sql(query, False, self.engine) if result: numeric_count, total_count = result[0] if numeric_count > total_count / 2: - self.numeric_cols.add(col) + self.numeric_mixed_cols.add(col) else: - self.categorical_cols.add(col) + self.categorical_mixed_cols.add(col) else: raise Exception(f"No rows fetched for column: {col} in table: {main_table_name}") def is_categorical_col(self, col_name: str): - return col_name in self.categorical_cols + return col_name in self.pure_categorical def is_numeric_col(self, col_name: str): """ @@ -136,3 +118,20 @@ def is_mixed_col(self, col_name: str): :return: whether the given col_name is of mixed type. """ return col_name in self.mixed_cols + + + def is_numeric_mixed_col(self, col_name: str): + """ + Determines whether the given column from the table used to construct this class is numeric among mixed types. + :arg: col_name: name of the column (assumes it is from the same table used to construct this class). + :return: whether the given col_name is majority numeric among mixed types. + """ + return col_name in self.numeric_mixed_cols + + def is_categorical_mixed_col(self, col_name: str): + """ + Determines whether the given column from the table used to construct this class is categorical among mixed types. + :arg: col_name: name of the column (assumes it is from the same table used to construct this class). + :return: whether the given col_name is majority categorical among mixed types. + """ + return col_name in self.categorical_mixed_cols diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index add561c..68a981b 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -5,6 +5,7 @@ from pandas.core.arrays import categorical from app.db_utils.execute_sql import fetch_sql +from app.db_utils.column_types import ColumnTypes # TODO: is this needed? this may be a duplicate def to_scalar(val): @@ -123,7 +124,7 @@ def calculate_summary_stat_using_sql(self, stat_query, column_name): # If its a numeric column with at least one string / categorical value, we only keep the numeric values so we # Can properly do calculations - if self.is_mixed_col(column_name): + if self.col_types.is_mixed_col(column_name): query += f' WHERE pg_input_is_valid("{column_name}", \'numeric\')' stat = fetch_sql(query, True, self.engine) @@ -178,7 +179,7 @@ def _calculate_median(self, column_name): # If its a numeric column with at least one string / categorical value, we only keep the numeric values so we # Can properly do calculations - if self.is_mixed_col(column_name): + if self.col_types.is_mixed_col(column_name): query += f' WHERE pg_input_is_valid("{column_name}", \'numeric\')' median = fetch_sql(query, True, self.engine) except Exception as e: diff --git a/app/db_utils/db_functions_sql.py b/app/db_utils/db_functions_sql.py index 86cc55f..9c07537 100644 --- a/app/db_utils/db_functions_sql.py +++ b/app/db_utils/db_functions_sql.py @@ -5,6 +5,7 @@ from app.server_utils import service_helpers from app.db_utils.filtering_sql import FilteringSQL from app.db_utils.execute_sql import fetch_sql, execute_sql +from app.db_utils.column_types import ColumnTypes """ Provides two classes for querying and visualizing data from a PostgreSQL database table, @@ -19,98 +20,6 @@ each annotated with per-bin/per-point error breakdowns. Also manages row-level data filters. """ -class ColumnTypes: - def __init__(self, main_table_name: str, engine): - self.numeric_cols = set() - self.categorical_mixed = set() - self.pure_categorical = set() - self.engine = engine - self.gather_numeric_cols(main_table_name) - self.gather_mixed_cols(main_table_name) - - - def gather_numeric_cols(self, main_table_name: str): - """ - Distinguishes the numeric columns from the categorical columns. - :arg: main_table_name: name of the main table. - """ - - - fetch_col_types = f'''SELECT column_name, data_type - FROM information_schema.columns - WHERE table_name = '{main_table_name}';''' - - fetched_rows = fetch_sql(fetch_col_types, False, self.engine) - if fetched_rows: - numeric_types = { - 'integer', 'bigint', 'numeric', - 'real', 'double precision', 'smallint' - } - - for row in fetched_rows: - col_name = row[0] - data_type = row[1] - - if data_type in numeric_types: - self.numeric_cols.add(col_name) - else: - self.categorical_mixed.add(col_name) - else: - raise Exception(f"No rows fetched from table: {main_table_name}") - - - def gather_mixed_cols(self, main_table_name: str): - """ - Gather the columns that are labeled as categorical but contain numeric data as well. - :arg: main_table_name: name of the main table. - """ - - # There are no categorical columns in the dataset. - if len(self.categorical_mixed) == 0: - return - - numeric_regex = r"'^\s*-?\d+(\.\d+)?\s*$'" - - # Initialized in the other constructor func gather_numeric_cols. This starts as all categorical columns. - # Stop early if a mixed type is found, since that makes the entire column of mixed type. - queries = [ - f"""( - SELECT '{col}' AS column_name - FROM "{main_table_name}" - WHERE "{col}" ~ {numeric_regex} - LIMIT 1 - )""" - for col in self.categorical_mixed - ] - - fetch_mixed_types = "\nUNION ALL\n".join(queries) - mixed_cols = fetch_sql(fetch_mixed_types, False, self.engine) - - mixed_col_names = set(row[0] for row in mixed_cols) if mixed_cols else set() - self.pure_categorical = self.categorical_mixed - mixed_col_names - self.categorical_mixed = mixed_col_names - - def is_categorical_col(self, col_name: str): - return col_name in self.pure_categorical - - def is_numeric_col(self, col_name: str): - """ - Determines whether the given column from the table used to construct this class is numeric. - :arg: col_name: name of the column (assumes it is from the same table used to construct this class). - :return: whether the given col_name is numeric. - """ - return col_name in self.numeric_cols - - - def is_mixed_col(self, col_name: str): - """ - Determines whether the given column from the table used to construct this class is of mixed type. - :arg: col_name: name of the column (assumes it is from the same table used to construct this class). - :return: whether the given col_name is of mixed type. - """ - return col_name in self.categorical_mixed - - # Wraps up all Core DBOperations into one class using a primary main_table. class DBOperations: def __init__(self, engine): @@ -123,6 +32,7 @@ def __init__(self, engine): self.engine = engine self.main_table_name = None self.error_table_name = None + self.dp_table_name = None self.col_types = None self.filtering_table = None self.active_hists = {} @@ -134,11 +44,12 @@ def reset(self): """ self.main_table_name = None self.error_table_name = None + self.dp_table_name = None self.col_types = None self.filtering_table = None self.active_hists = {} - def load_table(self, main_table_name: str, error_table_name: str = None): + def load_table(self, main_table_name: str, error_table_name: str = None, dp_table_name: str = None): """ Loads in the main and error tables, inits the ColumnTypes and FilteringSQL objects with the new table @@ -147,6 +58,7 @@ def load_table(self, main_table_name: str, error_table_name: str = None): """ self.main_table_name = main_table_name self.error_table_name = error_table_name if error_table_name is not None else "errors_" + main_table_name + self.dp_table_name = dp_table_name if dp_table_name is not None else "dp_" + main_table_name self.col_types = ColumnTypes(main_table_name, self.engine) self.filtering_table = FilteringSQL(main_table_name, self.engine) self.active_hists = {} diff --git a/app/routes/pgraph_routes.py b/app/routes/pgraph_routes.py index 4bbb39e..a0c152b 100644 --- a/app/routes/pgraph_routes.py +++ b/app/routes/pgraph_routes.py @@ -16,7 +16,7 @@ def set_selected_node(): body = request.get_json(force=True) clicked_node_id = body['nodeId'] current_table_name = clicked_node_access_helper(clicked_node_id) - db_operations.load_table(current_table_name, f"errors_{current_table_name}") + db_operations.load_table(current_table_name, f"errors_{current_table_name}", f"dp_{current_table_name}") return { "success": True, "current_table_name": current_table_name diff --git a/app/routes/plot_routes.py b/app/routes/plot_routes.py index e1e0ed1..c8c97cc 100644 --- a/app/routes/plot_routes.py +++ b/app/routes/plot_routes.py @@ -240,8 +240,9 @@ def get_preview_histogram(): try: errors_table = f"errors_{table}" + dp_table = f"dp_{table}" preview_ops = DBOperations(engine) - preview_ops.load_table(table, error_table_name=errors_table) + preview_ops.load_table(table, errors_table, dp_table) if type_ == "1d": column = request.args.get("column") @@ -283,8 +284,9 @@ def get_preview_scatterplot(): try: errors_table = f"errors_{table}" + dp_table = f"dp_{table}" preview_ops = DBOperations(engine) - preview_ops.load_table(table, error_table_name=errors_table) + preview_ops.load_table(table, errors_table, dp_table) scatterplot_data = preview_ops.generate_scatterplot_with_errors(x_column, y_column, error_sample_count, total_sample_count) return {"success": True, "scatterplot_data": scatterplot_data} except Exception as e: diff --git a/app/routes/routes.py b/app/routes/routes.py index 78968fb..3ef4b9f 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -41,6 +41,8 @@ def load_file(csv_file, filename): table_name_with_node_id = f"n0_{table_name}" # Build dtype map from actual column values before pushing to DB dtype_map = get_sqlalchemy_dtype_map(table_with_id_added) + error_table_name = f"errors_{table_name_with_node_id}" + dp_table_name = f"dp_{table_name_with_node_id}" try: """ @@ -52,20 +54,20 @@ def load_file(csv_file, filename): connectable which may not reflect the exact number of written rows as stipulated in the sqlite3 or SQLAlchemy. """ table_with_id_added.to_sql(table_name_with_node_id, engine, if_exists='replace', dtype=dtype_map) - detected_data.to_sql("errors_" + table_name_with_node_id, engine, if_exists='replace') + detected_data.to_sql(error_table_name, engine, if_exists='replace') - db_operations.load_table(table_name_with_node_id) + db_operations.load_table(table_name_with_node_id, error_table_name, dp_table_name) data_profile_df = create_data_profile_df(db_operations.data_profile) dtype_map = db_operations.data_profile.dtype_dict - data_profile_df.to_sql("dp_" + table_name_with_node_id, engine, if_exists='replace', dtype=dtype_map) + data_profile_df.to_sql(dp_table_name, engine, if_exists='replace', dtype=dtype_map) """ now we fully init the DBOperations object that was first initialized in init.py, get the actual row counts since .to_sql is buggy and not right """ rows_affected = db_operations.get_row_count(table_name_with_node_id) - detected_rows_affected = db_operations.get_row_count("errors_" + table_name_with_node_id) + detected_rows_affected = db_operations.get_row_count(error_table_name) #calculate the attribute rankings for the top 10 error rows table on the Buckaroo.tsx page rankings = calculate_attribute_rankings(detected_data) @@ -131,7 +133,7 @@ def undo_wrangle(): if not db_operations.table_exists(prev): return {"success": False, "error": f"Table '{prev}' does not exist"}, 404 - db_operations.load_table(prev, f"errors_{prev}") + db_operations.load_table(prev, f"errors_{prev}", f"dp_{prev}") return {"success": True, "table_name": prev} @@ -153,7 +155,7 @@ def redo_wrangle(): if next_node == db_operations.main_table_name: return {"success": False, "error": "You have reached the most up to date table"} - db_operations.load_table(next_node, f"errors_{next_node}") + db_operations.load_table(next_node, f"errors_{next_node}", f"dp_{next_node}") return {"success": True, "table_name": next_node} diff --git a/app/server_utils/data_attribute_summary_integration.py b/app/server_utils/data_attribute_summary_integration.py index e9ea92c..383d6ca 100644 --- a/app/server_utils/data_attribute_summary_integration.py +++ b/app/server_utils/data_attribute_summary_integration.py @@ -68,7 +68,7 @@ def get_attribute_stats(data_profile, column, main_df): :return: dictionary containing statistics for the column """ - if data_profile.is_categorical_col(column): + if data_profile.col_types.is_categorical_mixed_col(column): return get_categorical_stats(data_profile, column) return get_numeric_stats(data_profile, column) diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index 7295c44..c8171d6 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -209,8 +209,8 @@ def create_data_profile_df(data_profile, col_names=None): # Make sure that attribute and the column type match - numeric = (data_profile.is_numeric_col(col) and attribute in data_profile.attribute_type_assignment['numeric']) - categorical = (data_profile.is_categorical_col(col) and attribute in data_profile.attribute_type_assignment['categorical']) + numeric = ((data_profile.col_types.is_numeric_mixed_col(col) or data_profile.col_types.is_numeric_col )and attribute in data_profile.attribute_type_assignment['numeric']) + categorical = ((data_profile.is_categorical_mixed_col(col) or data_profile.col_types.is_categorical_col)and attribute in data_profile.attribute_type_assignment['categorical']) if not (numeric or categorical): From 5b9561857b8917fdbe697527428892b4246ffe91 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 20 Jul 2026 11:49:58 -0600 Subject: [PATCH 56/81] Fixed some bugs that I missed earlier --- app/routes/routes.py | 8 +++++--- app/server_utils/service_helpers.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/routes/routes.py b/app/routes/routes.py index 3ef4b9f..d65ad9c 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -7,6 +7,7 @@ import time from app import app from app import db_operations, engine +from app.db_utils.data_profile import DataProfile from app.server_utils.service_helpers import ( generate_table_name, create_error_df, @@ -55,10 +56,10 @@ def load_file(csv_file, filename): """ table_with_id_added.to_sql(table_name_with_node_id, engine, if_exists='replace', dtype=dtype_map) detected_data.to_sql(error_table_name, engine, if_exists='replace') + data_profile = DataProfile(table_name_with_node_id, engine) - db_operations.load_table(table_name_with_node_id, error_table_name, dp_table_name) - data_profile_df = create_data_profile_df(db_operations.data_profile) - dtype_map = db_operations.data_profile.dtype_dict + data_profile_df = create_data_profile_df(data_profile) + dtype_map = data_profile.dtype_dict data_profile_df.to_sql(dp_table_name, engine, if_exists='replace', dtype=dtype_map) @@ -66,6 +67,7 @@ def load_file(csv_file, filename): now we fully init the DBOperations object that was first initialized in init.py, get the actual row counts since .to_sql is buggy and not right """ + db_operations.load_table(table_name_with_node_id, error_table_name, dp_table_name) rows_affected = db_operations.get_row_count(table_name_with_node_id) detected_rows_affected = db_operations.get_row_count(error_table_name) diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index c8171d6..5698ca8 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -210,7 +210,7 @@ def create_data_profile_df(data_profile, col_names=None): # Make sure that attribute and the column type match numeric = ((data_profile.col_types.is_numeric_mixed_col(col) or data_profile.col_types.is_numeric_col )and attribute in data_profile.attribute_type_assignment['numeric']) - categorical = ((data_profile.is_categorical_mixed_col(col) or data_profile.col_types.is_categorical_col)and attribute in data_profile.attribute_type_assignment['categorical']) + categorical = ((data_profile.col_types.is_categorical_mixed_col(col) or data_profile.col_types.is_categorical_col)and attribute in data_profile.attribute_type_assignment['categorical']) if not (numeric or categorical): From 7a6eb4b0853cc59de7c93bfe90ee113be11a5bcf Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Thu, 23 Jul 2026 21:04:03 -0600 Subject: [PATCH 57/81] Modified _calculate_category_count_dict to not make the dict if all rows have a unique categorical value --- app/db_utils/data_profile.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index 68a981b..b360087 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -338,18 +338,30 @@ def _calculate_category_count_dict(self, column_name): """ try: - query = f""" - SELECT "{column_name}", COUNT(*) - FROM "{self.table_name}" - GROUP BY "{column_name}" - """ - rows = fetch_sql(query, False ,self.engine) - category_counts = {} - # Put the results into a dict - for (category, count) in rows: - category_counts[category] = count - print("CATEGORY COUNTS DICT", category_counts) + n_categories = self.calculate_column_attribute("n_categories", column_name, True) + n_rows_query = f""" + SELECT "{column_name}", COUNT(*) FROM "{self.table_name}" + """ + + n_rows = fetch_sql(n_rows_query, True, self.engine) + + # Don't wanna calculate category counts if each "category" is unique + if not n_categories == n_rows: + + query = f""" + SELECT "{column_name}", COUNT(*) + FROM "{self.table_name}" + GROUP BY "{column_name}" + """ + + rows = fetch_sql(query, False ,self.engine) + category_counts = {} + # Put the results into a dict + for (category, count) in rows: + category_counts[category] = count + else: + category_counts = None except Exception as e: From f6cb9cefbb89c755c7d6f24b37144c60bfe39c60 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Thu, 23 Jul 2026 21:04:42 -0600 Subject: [PATCH 58/81] Fixed bug where binned_data being None was causing an error. --- app/db_utils/db_functions_sql.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/db_utils/db_functions_sql.py b/app/db_utils/db_functions_sql.py index 9c07537..afd031e 100644 --- a/app/db_utils/db_functions_sql.py +++ b/app/db_utils/db_functions_sql.py @@ -152,6 +152,10 @@ def update_active_hists(self, binned_data: list, one_dim: bool, col_key): rows_to_bins = {} bins_to_rows = defaultdict(list) + if binned_data is None: + # Will happen if all of the data happens to be deleted (all values are Null in a column, etc.) + # Setting to empty list so it won't show error when it tries to loop through a None + binned_data = [] if one_dim: for row_id, row_bin in binned_data: From 48dd184720665cda689b7a68e9aab5c6c9a8bc99 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 24 Jul 2026 08:56:51 -0600 Subject: [PATCH 59/81] Added debugging print statements --- app/db_utils/db_functions_sql.py | 1 + app/routes/plot_routes.py | 1 + 2 files changed, 2 insertions(+) diff --git a/app/db_utils/db_functions_sql.py b/app/db_utils/db_functions_sql.py index afd031e..4112905 100644 --- a/app/db_utils/db_functions_sql.py +++ b/app/db_utils/db_functions_sql.py @@ -62,6 +62,7 @@ def load_table(self, main_table_name: str, error_table_name: str = None, dp_tabl self.col_types = ColumnTypes(main_table_name, self.engine) self.filtering_table = FilteringSQL(main_table_name, self.engine) self.active_hists = {} + print("LOADED TABLE!!!") def get_row_count(self, table_name: str) -> int: """ diff --git a/app/routes/plot_routes.py b/app/routes/plot_routes.py index c8c97cc..10a565f 100644 --- a/app/routes/plot_routes.py +++ b/app/routes/plot_routes.py @@ -255,6 +255,7 @@ def get_preview_histogram(): x_bins = int(request.args.get("x_bins", 10)) y_bins = int(request.args.get("y_bins", 10)) histogram = preview_ops.generate_two_d_histogram_with_errors(column_x, column_y, x_bins, y_bins) + print("PREVIEW OPS FILTERING TABLE!!!!", preview_ops.filtering_table) return {"success": True, "histogram": histogram} except Exception as e: From 8b57dcde62c33d6eefc9e4ae374a7c28d2884b55 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 24 Jul 2026 09:03:22 -0600 Subject: [PATCH 60/81] Added base_table_name variable to DBOperations. Refactored generate_table_name to generate_base_table_name --- app/db_utils/db_functions_sql.py | 11 ++++++++++- app/server_utils/service_helpers.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/app/db_utils/db_functions_sql.py b/app/db_utils/db_functions_sql.py index 4112905..3309aab 100644 --- a/app/db_utils/db_functions_sql.py +++ b/app/db_utils/db_functions_sql.py @@ -30,6 +30,7 @@ def __init__(self, engine): :param engine: SQLAlchemy engine """ self.engine = engine + self.base_table_name = None self.main_table_name = None self.error_table_name = None self.dp_table_name = None @@ -42,6 +43,7 @@ def reset(self): Resets the DBOperations state, clearing all loaded table references. Called when the user navigates back to the home page. """ + self.base_table_name = None self.main_table_name = None self.error_table_name = None self.dp_table_name = None @@ -49,21 +51,28 @@ def reset(self): self.filtering_table = None self.active_hists = {} - def load_table(self, main_table_name: str, error_table_name: str = None, dp_table_name: str = None): + def load_table(self, main_table_name: str, error_table_name: str = None, dp_table_name: str = None, base_table_name: str = None): """ Loads in the main and error tables, inits the ColumnTypes and FilteringSQL objects with the new table :param main_table_name: the name of the table in the database without errors detected (raw data) :param error_table_name: explicit errors table name; defaults to "errors_" + main_table_name """ + self.main_table_name = main_table_name self.error_table_name = error_table_name if error_table_name is not None else "errors_" + main_table_name self.dp_table_name = dp_table_name if dp_table_name is not None else "dp_" + main_table_name self.col_types = ColumnTypes(main_table_name, self.engine) self.filtering_table = FilteringSQL(main_table_name, self.engine) + assert self.filtering_table is not None self.active_hists = {} print("LOADED TABLE!!!") + if base_table_name is not None: + self.base_table_name = base_table_name + + print("FINISHED LOADING TABLE") + def get_row_count(self, table_name: str) -> int: """ Returns the actual row count of a table directly from the database. diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index 5698ca8..77ae284 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -70,7 +70,7 @@ def _safe_pg_name(base: str, suffix: str) -> str: return f"{base[:max_base]}_{h}{suffix}" -def generate_table_name(csv_name): +def generate_base_table_name(csv_name): """ Cleans the file name so that it is ready to be used to make a table in the database, it needs to: - Remove file extension (.csv), replace spaces/special chars with underscores, ensure it starts with a letter (SQL requirement) From 359c4c04606ad4903e39f83e8eec46467fb5abe4 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 24 Jul 2026 09:04:10 -0600 Subject: [PATCH 61/81] Implemented logger_utils.py (initialize_user_log and update_user_log functions) --- app/server_utils/logger_utils.py | 63 ++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 app/server_utils/logger_utils.py diff --git a/app/server_utils/logger_utils.py b/app/server_utils/logger_utils.py new file mode 100644 index 0000000..1fa918f --- /dev/null +++ b/app/server_utils/logger_utils.py @@ -0,0 +1,63 @@ +import uuid +from flask import request +import pandas as pd +import traceback + +from datetime import datetime, timezone +from sqlalchemy import Text, TIMESTAMP +from sqlalchemy.dialects.postgresql import UUID # if using Postgres +import logging +import json +logger = logging.getLogger(__name__) + +ACTION_LOG_TABLE_NAME = "action_log" + +# TODO: this definitely needs to be condensed into a single functon (initializing __ df) +# Just making this as a df to match the format of the other table creation functions +# TODO: switch all of the sql table creation functions to not use pandas because apparently it's very slow +def create_empty_user_action_log_df(): + empty_df = pd.DataFrame(columns=['action_id', 'dataset_id', 'action_name', 'action_details', 'timestamp']) + + return empty_df + + +# TODO: update documentation +def update_action_log(dataset_id, action_name, action_details, engine): + try: + timestamp = datetime.now(timezone.utc) + + if action_details is not None: + action_details = json.dumps(action_details) + + # Create an action id + action_id = uuid.uuid4() + + new_action_entry = pd.DataFrame([{"action_id": action_id, "dataset_id": dataset_id, "action_name": action_name, "action_details": action_details, "timestamp": timestamp}]) + new_action_entry.to_sql(ACTION_LOG_TABLE_NAME, engine, if_exists='append', index=False) + except Exception: + logger.error("Error updating action log.", exc_info=True) + + + +def initialize_action_log(engine, reset_log=False): + print("INITIALIZING ACTION LOG") + dtype_map = { + 'action_id': UUID(as_uuid=True), + 'dataset_id': Text, + 'action_name': Text, + 'action_details': Text, + 'timestamp': TIMESTAMP(timezone=True) + } + + try: + empty_log_df = create_empty_user_action_log_df() + + # When we actually can support multiple users, make this name to be user / session specific + if reset_log: + empty_log_df.to_sql(ACTION_LOG_TABLE_NAME, engine, if_exists='replace', index=False, dtype=dtype_map) + else: + empty_log_df.to_sql(ACTION_LOG_TABLE_NAME, engine, if_exists='append', index=False, dtype=dtype_map) + except Exception: + logger.error("Error initializing action log.", exc_info=True) + + From 5abbb744f9ec71868d7ceb4692acb0e17f703bbd Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 24 Jul 2026 09:05:14 -0600 Subject: [PATCH 62/81] Implemented logger_utils.py (initialize_user_log and update_user_log functions) --- app/routes/routes.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/app/routes/routes.py b/app/routes/routes.py index d65ad9c..639701a 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -9,7 +9,7 @@ from app import db_operations, engine from app.db_utils.data_profile import DataProfile from app.server_utils.service_helpers import ( - generate_table_name, + generate_base_table_name, create_error_df, get_sqlalchemy_dtype_map, calculate_attribute_rankings, get_pgraph_redo, get_pgraph_undo, init_pgraph_for_session, create_data_profile_df, @@ -38,8 +38,12 @@ def load_file(csv_file, filename): detected_data = create_error_df(dataframe) time_to_detect = time.time() - start_time app.original_table_name = filename - table_name = generate_table_name(filename) - table_name_with_node_id = f"n0_{table_name}" + base_table_name = generate_base_table_name(filename) + + #initialize_action_log(engine) + #update_action_log(dataset_id=base_table_name, action_name="load_dataset", action_details=None, engine=engine) + + table_name_with_node_id = f"n0_{base_table_name}" # Build dtype map from actual column values before pushing to DB dtype_map = get_sqlalchemy_dtype_map(table_with_id_added) error_table_name = f"errors_{table_name_with_node_id}" @@ -67,7 +71,8 @@ def load_file(csv_file, filename): now we fully init the DBOperations object that was first initialized in init.py, get the actual row counts since .to_sql is buggy and not right """ - db_operations.load_table(table_name_with_node_id, error_table_name, dp_table_name) + db_operations.load_table(table_name_with_node_id, error_table_name, dp_table_name, base_table_name=base_table_name) + print("DB OPERATIONS LOAD_TABLE DONE") rows_affected = db_operations.get_row_count(table_name_with_node_id) detected_rows_affected = db_operations.get_row_count(error_table_name) From 1c2a2b8570aa9c1325e2b03c080c3132e3283beb Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 24 Jul 2026 10:10:05 -0600 Subject: [PATCH 63/81] Added calls to initialize and update action logs (update when dataset is loaded, preview is created, and wrangle is executed --- app/routes/routes.py | 5 +++-- app/server_utils/service_helpers.py | 14 ++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/app/routes/routes.py b/app/routes/routes.py index 639701a..9ab2caf 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -15,6 +15,7 @@ calculate_attribute_rankings, get_pgraph_redo, get_pgraph_undo, init_pgraph_for_session, create_data_profile_df, ) from app.server_utils.set_id_column import set_id_column +from app.server_utils.logger_utils import update_action_log, initialize_action_log def load_file(csv_file, filename): @@ -40,8 +41,8 @@ def load_file(csv_file, filename): app.original_table_name = filename base_table_name = generate_base_table_name(filename) - #initialize_action_log(engine) - #update_action_log(dataset_id=base_table_name, action_name="load_dataset", action_details=None, engine=engine) + initialize_action_log(engine) + update_action_log(dataset_id=base_table_name, action_name="load_dataset", action_details=None, engine=engine) table_name_with_node_id = f"n0_{base_table_name}" # Build dtype map from actual column values before pushing to DB diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index 77ae284..820bb2f 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -19,7 +19,7 @@ from detectors.datatype_mismatch import datatype_mismatch from detectors.incomplete import incomplete from detectors.missing_value import missing_value -from app.db_utils.data_profile import DataProfile +from app.server_utils.logger_utils import update_action_log def get_current_pgraph(): """ @@ -218,8 +218,6 @@ def create_data_profile_df(data_profile, col_names=None): continue - print("CALCULATING ATTRIBUTE: ", attribute) - print("COLUMN: ", col) row_dict[attribute] = data_profile.calculate_column_attribute(attribute, col, False) col_list.append(row_dict) @@ -420,7 +418,6 @@ def execute_wrangle_preview(table, preview_table, safe_pg_name_fn, db_operations 3. Reload db_operations with the new node Returns a dict with success and table name. """ - # from app import engine, db_operations all_possible_previews = [ safe_pg_name_fn(table, "_preview_delete"), @@ -442,6 +439,9 @@ def execute_wrangle_preview(table, preview_table, safe_pg_name_fn, db_operations app.db_operations.update_rankings(new_table_name) + update_action_log(dataset_id=db_operations.base_table_name, action_name=f"{wrangle_executed}_wrangle", + action_details={}, engine=db_operations.engine) + return {"success": True, "table": new_table_name} @@ -536,6 +536,9 @@ def create_previews_1d(table, row_ids, cols, safe_pg_name_fn, update_errors_fn, update_data_profile_table_fn(preview_delete_table_name, cols) update_data_profile_table_fn(preview_impute_table_name, cols) + update_action_log(dataset_id=table, action_name="create_previews", + action_details=json.dumps({"row_ids": row_ids, "cols": cols}), engine=engine) + return { "success": True, "preview_delete": preview_delete_table_name, @@ -581,6 +584,9 @@ def create_previews_2d(table, row_ids, cols, safe_pg_name_fn, update_errors_fn, update_data_profile_table_fn(preview_impute_x_table_name, cols) update_data_profile_table_fn(preview_impute_y_table_name, cols) + update_action_log(dataset_id=table, action_name="create_previews", + action_details=json.dumps({"row_ids": row_ids, "cols": cols}), engine=engine) + return { "success": True, From 16b9ab96b5dcdb720c83a527028e034b0631c94f Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 24 Jul 2026 10:15:34 -0600 Subject: [PATCH 64/81] WIP: Adding update_action_log to deleting column action --- app/routes/wrangler_routes_sql.py | 35 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index 5e5fe8b..a316b85 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -2,9 +2,8 @@ # This file handles all endpoints surrounding wranglers from flask import request -from app import app, db_operations from app.db_utils import query -from app import engine +from app import app, engine, db_operations import traceback import pandas as pd from app.server_utils.service_helpers import create_error_df, create_previews_1d, create_previews_2d, \ @@ -12,17 +11,12 @@ from sqlalchemy import inspect, text from app.db_utils.data_profile import DataProfile +from server_utils.logger_utils import update_action_log """ Wrangling Endpoints - In-place modification of tables """ -def get_table_dtypes(target_table_name, engine): - """Build a dtype dict for to_sql() by reflecting the target table's real column types.""" - inspector = inspect(engine) - columns = inspector.get_columns(target_table_name) - # col["type"] is already a SQLAlchemy type instance we can hand straight to to_sql - return {col["name"]: col["type"] for col in columns} # Where updated_df is just the data that needed to actually be updated # Assumes that updated_df has the same columns as the target table @@ -59,6 +53,17 @@ def update_table(updated_df, target_table_name, key_col, cols_to_remove): # Helper: Re-run error detection after modification # ───────────────────────────────────────────────────────────────────────────── +#def mark_dirty_rows_data_profile(table_name, col_names): + + +#def mark_dirty_rows_errors_table(table_name, col_names): + + +# TODO: Finish this later +#def update_table_rows(table_name, col_names: list) -> None: + + + # Returns error_df for update_data_profile_table to use (so it doesn't have to get it from the database) def update_errors_table(table_name: str, columns_selected_for_wrangling: list) -> pd.DataFrame: # TODO: fix this so it doesn't update the whole table after small changes to the table @@ -87,7 +92,6 @@ def update_errors_table(table_name: str, columns_selected_for_wrangling: list) - # detected_errors_df.to_sql(errors_table_name, engine, if_exists='fail', index=False) print(f"✓ Updated errors table: {errors_table_name}") - return detected_errors_df except Exception as e: print(f"ERROR: Could not update errors table for {table_name}: {e}") traceback.print_exc() @@ -120,15 +124,6 @@ def update_data_profile_table(table_name: str, columns_selected_for_wrangling: l raise -# TODO: Finish this later -#def update_stat_to_data_profile_table(table_name: str, error_df: pd.DataFrame) -> None: - - - - - - -# TODO: does this even do anything? Can I remove it? def update_preview_error_table(table_name: str, err_table_name: str) -> None: """ After modifying a table in-place, re-run error detection @@ -195,6 +190,7 @@ def create_previews(): else: return create_previews_2d(table, row_ids, cols, _safe_pg_name, update_errors_table, update_data_profile_table) + except Exception as e: print("ERROR in create_previews") print(traceback.format_exc()) @@ -222,8 +218,10 @@ def execute_wrangle(): return {"success": False, "error": str(e)}, 400 +# TODO: check if the column delete functionality actually even works @app.post("/api/wrangle/delete-column") def wrangle_delete_column(): + # TODO: why doesn't this have versioning? What if the user wants to undo this action? """ Delete a column from the table in-place. @@ -242,6 +240,7 @@ def wrangle_delete_column(): # Re-run error detection update_errors_table(table_name, [column]) update_data_profile_table(table_name, [column]) + update_action_log(dataset_id=db_operations.base_table_name,action_name="delete_column", action_details={"column": column}, engine=engine) return { "success": True, From 30e181fa1bd13ff80d1f2a3840cc5175bba1f534 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 31 Jul 2026 12:58:23 -0600 Subject: [PATCH 65/81] Created logger utils functions --- app/server_utils/logger_utils.py | 81 ++++++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 8 deletions(-) diff --git a/app/server_utils/logger_utils.py b/app/server_utils/logger_utils.py index 1fa918f..a98074c 100644 --- a/app/server_utils/logger_utils.py +++ b/app/server_utils/logger_utils.py @@ -4,36 +4,46 @@ import traceback from datetime import datetime, timezone -from sqlalchemy import Text, TIMESTAMP +from sqlalchemy import Text, TIMESTAMP, Boolean, Float from sqlalchemy.dialects.postgresql import UUID # if using Postgres import logging import json + +from app.db_utils.execute_sql import fetch_sql + logger = logging.getLogger(__name__) ACTION_LOG_TABLE_NAME = "action_log" +PREVIEW_LOG_TABLE_NAME = "preview_log" -# TODO: this definitely needs to be condensed into a single functon (initializing __ df) -# Just making this as a df to match the format of the other table creation functions # TODO: switch all of the sql table creation functions to not use pandas because apparently it's very slow def create_empty_user_action_log_df(): - empty_df = pd.DataFrame(columns=['action_id', 'dataset_id', 'action_name', 'action_details', 'timestamp']) + empty_df = pd.DataFrame(columns=['action_id', 'dataset_id', 'action_name', 'action_details', 'timestamp', 'action_duration', 'action_successful', 'action_error_message']) + return empty_df +def create_empty_preview_log_df(): + empty_df = pd.DataFrame(columns=['preview_table_name', 'action_name', 'action_details']) return empty_df # TODO: update documentation -def update_action_log(dataset_id, action_name, action_details, engine): +def update_action_log(dataset_id, action_name, action_details, engine, timestamp, action_successful, + action_duration=None, action_error_message=None): try: - timestamp = datetime.now(timezone.utc) if action_details is not None: action_details = json.dumps(action_details) # Create an action id action_id = uuid.uuid4() + print("ACTION DURATION TYPE:", type(action_duration)) + + new_action_entry = pd.DataFrame([{"action_id": action_id, "dataset_id": dataset_id, "action_name": action_name, + "action_details": action_details, "timestamp": timestamp, "action_duration": action_duration, + "action_successful": action_successful, 'action_error_message': action_error_message}]) - new_action_entry = pd.DataFrame([{"action_id": action_id, "dataset_id": dataset_id, "action_name": action_name, "action_details": action_details, "timestamp": timestamp}]) new_action_entry.to_sql(ACTION_LOG_TABLE_NAME, engine, if_exists='append', index=False) + print("UPDATED ACTION LOG TABLE") except Exception: logger.error("Error updating action log.", exc_info=True) @@ -46,7 +56,10 @@ def initialize_action_log(engine, reset_log=False): 'dataset_id': Text, 'action_name': Text, 'action_details': Text, - 'timestamp': TIMESTAMP(timezone=True) + 'timestamp': TIMESTAMP, + 'action_duration': Float, + 'action_successful': Boolean, + 'action_error_message': Text } try: @@ -60,4 +73,56 @@ def initialize_action_log(engine, reset_log=False): except Exception: logger.error("Error initializing action log.", exc_info=True) +def initialize_preview_log_table(engine, reset_log=False): + print("INITIALIZING PREVIEW LOG TABLE") + dtype_map = { + 'preview_table_name':Text, + 'action_name': Text, + 'action_details': Text + } + + try: + empty_log_df = create_empty_preview_log_df() + + if reset_log: + empty_log_df.to_sql(PREVIEW_LOG_TABLE_NAME, engine, if_exists='replace', index=False, dtype=dtype_map) + else: + empty_log_df.to_sql(PREVIEW_LOG_TABLE_NAME, engine, if_exists='append', index=False, dtype=dtype_map) + print("INITIALIZED PREVIEW LOG TABLE") + except Exception: + logger.error("Error initializing preview log table.", exc_info=True) + + +def update_preview_log(preview_table_name, action_name, action_details, engine): + try: + + if action_details is not None: + action_details = json.dumps(action_details) + + new_action_entry = pd.DataFrame([{"preview_table_name": preview_table_name, "action_name": action_name, "action_details": action_details}]) + new_action_entry.to_sql(PREVIEW_LOG_TABLE_NAME, engine, if_exists='append', index=False) + print("UPDATED PREVIEW LOG TABLE") + except Exception: + logger.error("Error updating preview log table.", exc_info=True) + +def get_action_details_from_preview_log(preview_table_name, engine): + try: + + query = f""" + SELECT action_details + FROM "{PREVIEW_LOG_TABLE_NAME}" + WHERE preview_table_name = :id + """ + + result = fetch_sql(query, params={"id": preview_table_name},scalar=True, engine=engine) + return result + except Exception: + logger.error(f"Error retrieving action details from {preview_table_name} from preview log.", exc_info=True) + result = None + return result + + + + + From 6c2966817dbc08682503a32211ee0ea54f4670a2 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 31 Jul 2026 12:59:25 -0600 Subject: [PATCH 66/81] Added logging functions to API calls and moved some logging function calls to the proper locations --- app/routes/routes.py | 14 ++++- app/routes/wrangler_routes_sql.py | 89 +++++++++++++++++++++++++++-- app/server_utils/service_helpers.py | 25 ++------ 3 files changed, 100 insertions(+), 28 deletions(-) diff --git a/app/routes/routes.py b/app/routes/routes.py index 9ab2caf..99c8546 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -14,8 +14,9 @@ get_sqlalchemy_dtype_map, calculate_attribute_rankings, get_pgraph_redo, get_pgraph_undo, init_pgraph_for_session, create_data_profile_df, ) +from datetime import datetime, timezone from app.server_utils.set_id_column import set_id_column -from app.server_utils.logger_utils import update_action_log, initialize_action_log +from app.server_utils.logger_utils import update_action_log, initialize_action_log, initialize_preview_log_table def load_file(csv_file, filename): @@ -31,6 +32,7 @@ def load_file(csv_file, filename): :param filename: the name of the csv_file :return: json object """ + timestamp = datetime.now(timezone.utc) dataframe = pd.read_csv(csv_file) # run the detectors on the uploaded file for the starting data state @@ -42,7 +44,8 @@ def load_file(csv_file, filename): base_table_name = generate_base_table_name(filename) initialize_action_log(engine) - update_action_log(dataset_id=base_table_name, action_name="load_dataset", action_details=None, engine=engine) + initialize_preview_log_table(engine) + table_name_with_node_id = f"n0_{base_table_name}" # Build dtype map from actual column values before pushing to DB @@ -83,13 +86,20 @@ def load_file(csv_file, filename): #init the pgraph init_pgraph_for_session(table_name_with_node_id) + action_duration = datetime.now(timezone.utc) - timestamp + update_action_log(dataset_id=base_table_name, action_name="load_dataset", action_details=None, engine=engine, + timestamp=timestamp, action_duration=action_duration, action_successful=True) return {"success": True, "rows for undetected data": rows_affected, "rows_for_detected": detected_rows_affected, "table_name": table_name_with_node_id} except Exception as e: print(f"Error in upload: {e}") import traceback traceback.print_exc() + + update_action_log(dataset_id=base_table_name, action_name="load_dataset", action_details=None, engine=engine, + timestamp=timestamp, action_successful=False, + action_error_message=e) return {"success": False, "error": str(e)} diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index a316b85..af5f962 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -7,11 +7,13 @@ import traceback import pandas as pd from app.server_utils.service_helpers import create_error_df, create_previews_1d, create_previews_2d, \ - execute_wrangle_preview, _safe_pg_name, create_data_profile_df, get_sqlalchemy_dtype_map + execute_wrangle_preview, _safe_pg_name, create_data_profile_df, get_sqlalchemy_dtype_map, extract_preview_action from sqlalchemy import inspect, text from app.db_utils.data_profile import DataProfile -from server_utils.logger_utils import update_action_log +from datetime import datetime, timezone +from app.server_utils.logger_utils import update_action_log, update_preview_log, get_action_details_from_preview_log +import json """ Wrangling Endpoints - In-place modification of tables @@ -173,25 +175,74 @@ def create_previews(): row_ids – list of integer row IDs to operate on cols – list of column names involved in the selection (for imputation) """ + + timestamp = datetime.now(timezone.utc) try: body = request.get_json(force=True) table = db_operations.main_table_name row_ids = body.get("row_ids", []) cols = body.get("cols", []) + # extra case protection. #cols = [f'{col}' for col in cols] if not row_ids: + update_action_log(dataset_id=table, action_name="create_previews", + action_details=json.dumps({"row_ids": row_ids, "cols": cols}), engine=engine, + timestamp=timestamp, action_successful=False, action_error_message="Row IDs list is empty") + return {"success": False, "error": "No rows selected"}, 400 + + action_details_dict = {"row_ids": row_ids, "cols": cols} + if len(cols) == 1: - return create_previews_1d(table, row_ids, cols, _safe_pg_name, update_errors_table, update_data_profile_table) + + (preview_delete_table_name, preview_impute_table_name) = create_previews_1d(table, row_ids, cols, _safe_pg_name, update_errors_table, update_data_profile_table) + + update_preview_log(preview_delete_table_name, "delete_wrangle", action_details_dict, engine) + update_preview_log(preview_impute_table_name, "impute_wrangle", action_details_dict, engine) + + result_dict = { + "success": True, + "preview_delete": preview_delete_table_name, + "preview_impute": preview_impute_table_name, + "dims": 1, + } else: - return create_previews_2d(table, row_ids, cols, _safe_pg_name, update_errors_table, update_data_profile_table) + (preview_delete_table_name, preview_impute_x_table_name, preview_impute_y_table_name) = create_previews_2d(table, row_ids, cols, _safe_pg_name, update_errors_table, update_data_profile_table) + + update_preview_log(preview_delete_table_name, "delete_wrangle", action_details_dict, engine) + update_preview_log(preview_impute_x_table_name, "impute_x_wrangle", action_details_dict, engine) + update_preview_log(preview_impute_y_table_name, "impute_y_wrangle", action_details_dict, engine) + + result_dict = { + "success": True, + "preview_delete": preview_delete_table_name, + "preview_impute_x": preview_impute_x_table_name, + "preview_impute_y": preview_impute_y_table_name, + "dims": 2, + } + + action_duration = datetime.now(timezone.utc) - timestamp + action_duration_seconds = action_duration.total_seconds() + + update_action_log(dataset_id=table, action_name="create_previews", + action_details=json.dumps(action_details_dict), engine=engine, + timestamp=timestamp, action_duration= action_duration_seconds,action_successful=True) + + assert result_dict is not None + + return result_dict except Exception as e: + + update_action_log(dataset_id=table, action_name="create_previews", + action_details=json.dumps({"row_ids": row_ids, "cols": cols}), engine=engine, + timestamp=timestamp, action_successful=False, action_error_message=e) + print("ERROR in create_previews") print(traceback.format_exc()) return {"success": False, "error": str(e)}, 400 @@ -206,15 +257,30 @@ def execute_wrangle(): 3. Rename the selected preview table to 4. Delete
_old """ + timestamp = datetime.now(timezone.utc) try: body = request.get_json(force=True) table = db_operations.main_table_name preview_table = body["preview_table"] # the preview to promote + action_details_dict = get_action_details_from_preview_log(preview_table, engine) + + wrangle_executed = extract_preview_action(preview_table) + + new_table_name = execute_wrangle_preview(table, preview_table, _safe_pg_name, db_operations) - return execute_wrangle_preview(table, preview_table, _safe_pg_name, db_operations) + # TODO: incorporate action details from preview log table into this + action_duration = datetime.now(timezone.utc) - timestamp + action_duration_seconds = action_duration.total_seconds() + update_action_log(dataset_id=db_operations.base_table_name, action_name=f"{wrangle_executed}_wrangle", + action_details=action_details_dict, engine=db_operations.engine, timestamp=timestamp, action_duration= action_duration_seconds,action_successful=True) + + return {"success": True, "table": new_table_name} except Exception as e: print("ERROR in execute_wrangle") print(traceback.format_exc()) + + update_action_log(dataset_id=db_operations.base_table_name, action_name=f"{wrangle_executed}_wrangle", + action_details=action_details_dict, engine=db_operations.engine, timestamp=timestamp, action_duration= None,action_successful=False, action_error_message=e) return {"success": False, "error": str(e)}, 400 @@ -227,6 +293,8 @@ def wrangle_delete_column(): Modifies the table directly - no versioning. """ + + timestamp = datetime.now(timezone.utc) try: body = request.get_json(force=True) table_name = db_operations.main_table_name @@ -240,7 +308,12 @@ def wrangle_delete_column(): # Re-run error detection update_errors_table(table_name, [column]) update_data_profile_table(table_name, [column]) - update_action_log(dataset_id=db_operations.base_table_name,action_name="delete_column", action_details={"column": column}, engine=engine) + action_duration = datetime.now(timezone.utc) - timestamp + action_duration_seconds = action_duration.total_seconds() + update_action_log(dataset_id=db_operations.base_table_name, action_name="delete_column", + action_details={"column": column}, engine=engine, timestamp=timestamp, action_duration= action_duration_seconds, + action_successful=True) + return { "success": True, @@ -250,4 +323,8 @@ def wrangle_delete_column(): except Exception as e: print("ERROR OCCURRED") print(traceback.format_exc()) + + update_action_log(dataset_id=db_operations.base_table_name, action_name=f"delete_column", + action_details={"column": column}, engine=db_operations.engine, timestamp=timestamp, action_duration= None,action_successful=False, action_error_message=e) + return {"success": False, "error": str(e)}, 400 diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index 820bb2f..f97937f 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -20,6 +20,8 @@ from detectors.incomplete import incomplete from detectors.missing_value import missing_value from app.server_utils.logger_utils import update_action_log +from datetime import datetime, timezone +import logging def get_current_pgraph(): """ @@ -439,11 +441,9 @@ def execute_wrangle_preview(table, preview_table, safe_pg_name_fn, db_operations app.db_operations.update_rankings(new_table_name) - update_action_log(dataset_id=db_operations.base_table_name, action_name=f"{wrangle_executed}_wrangle", - action_details={}, engine=db_operations.engine) + return new_table_name - return {"success": True, "table": new_table_name} def _clone_table_pair(conn, source_table, dest_table, errors_source, dp_source): """Drop-and-recreate dest_table and its errors_ and dp_ sibling as copies of source tables.""" @@ -536,15 +536,8 @@ def create_previews_1d(table, row_ids, cols, safe_pg_name_fn, update_errors_fn, update_data_profile_table_fn(preview_delete_table_name, cols) update_data_profile_table_fn(preview_impute_table_name, cols) - update_action_log(dataset_id=table, action_name="create_previews", - action_details=json.dumps({"row_ids": row_ids, "cols": cols}), engine=engine) + return (preview_delete_table_name, preview_impute_table_name) - return { - "success": True, - "preview_delete": preview_delete_table_name, - "preview_impute": preview_impute_table_name, - "dims": 1, - } def extract_preview_action(name: str) -> str: """Extract the action after '_preview_' (e.g. 'impute_y'), or '' if not found.""" @@ -584,17 +577,9 @@ def create_previews_2d(table, row_ids, cols, safe_pg_name_fn, update_errors_fn, update_data_profile_table_fn(preview_impute_x_table_name, cols) update_data_profile_table_fn(preview_impute_y_table_name, cols) - update_action_log(dataset_id=table, action_name="create_previews", - action_details=json.dumps({"row_ids": row_ids, "cols": cols}), engine=engine) + return (preview_delete_table_name, preview_impute_x_table_name, preview_impute_y_table_name) - return { - "success": True, - "preview_delete": preview_delete_table_name, - "preview_impute_x": preview_impute_x_table_name, - "preview_impute_y": preview_impute_y_table_name, - "dims": 2, - } def _parse_node_id(table_name): """Parse 'n3_rest_of_name' into (3, 'rest_of_name'). Returns None on failure.""" From a05a873c6b1c6d5b566c05f1b7a4d34fa87664be Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 7 Aug 2026 16:29:39 -0600 Subject: [PATCH 67/81] Created ablation_study.py --- app/ablation_study/ablation_study.py | 126 +++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 app/ablation_study/ablation_study.py diff --git a/app/ablation_study/ablation_study.py b/app/ablation_study/ablation_study.py new file mode 100644 index 0000000..fd9d21d --- /dev/null +++ b/app/ablation_study/ablation_study.py @@ -0,0 +1,126 @@ +import os + +from dotenv import load_dotenv +import datetime + + +from app import app as app_module +from app import db_operations, engine + +import json + +from app.db_utils.ai_utils import parse_json_response +from app.server_utils.logger_utils import initialize_action_log + +datasets_paths = ['provided_datasets/mari_dataset.csv'] +#models = [{"model": "qwen/qwen3.6-27b", "provider": "groq"}] +models = [{"model": "openai/gpt-oss-20b", "provider": "groq"}] + +def variant(name, **overrides): + baseline = { + "name": "baseline", + "include_error_log": True, + "include_data_profile": True, + + "include_action_log": True, + "action_log_limit": 10, + "include_full_dataset": True + } + + # gets the baseline and overrides the key(s) in the overrides variable + config = {**baseline, **overrides} + config["name"] = name + return config + + +def is_stop_action(action_name): + if action_name == "stop": + return True + else: + return False + + +if __name__ == "__main__": + load_dotenv() + + # TODO: double check this + ablation_configs = [ + variant("baseline"), + variant("no_error_log", include_error_log=False), + variant("no_data_profile", include_error_log=False), + variant("no_action_log", include_error_log=False), + variant("no_full_dataset", include_error_log=False), + variant("no_action_log_limit", action_log_limit=None) # Includes full action log + ] + + client = app_module.test_client() + app_module.testing = False + + results = [] + + for model_dict in models: + + model = model_dict["model"] + model_provider = model_dict["provider"] + + update_settings_table_result = client.post('/api/ai_helper/update_settings_table', json={"model_name": model, "provider": model_provider}) + data = update_settings_table_result.get_json() + assert data["success"] == True + + for config in ablation_configs: + # reset globals living in the app package namespace + app_module.wrangle_occurred = False + + # reset attributes on the Flask object itself + app_module.pgraph_for_session = None + + # reset your stateful class instance + db_operations.reset() + for dataset in datasets_paths: + initialize_action_log(engine, reset_log=True) + + with open(dataset, 'rb') as f: + upload_result = client.post('/api/upload', data={'file': (f, dataset)}, + content_type='multipart/form-data') + data = upload_result.get_json() + assert data["success"] == True + + result_dict = {} # Dict added to the result json + result_dict["model"] = model_dict + result_dict["config_name"] = config["name"] + result_dict["dataset"] = dataset + result_dict["actions"] = [] + + action_plan_batch = 0 + + stop_action_found = False + + while not stop_action_found: + action_plan_batch += 1 + + actions = None + + # Get actions + get_action_plan_result = client.post('/api/ai_helper/get_action_plan') + action_plan_result_json = get_action_plan_result.get_json() + assert action_plan_result_json["success"] == True + action_plan_json = action_plan_result_json["json_action_plan"] + + # Go through all actions and perform each of them + for action_dict in action_plan_json: + + if is_stop_action(action_dict["action_name"]): + stop_action_found = True + + action_result = client.post('/api/ai_helper/perform_llm_action', json=action_dict) + action_dict["action_plan_batch"] = action_plan_batch + action_dict["timestamp"] = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + action_dict["success"] = action_result.get_json() + + results.append(action_dict) + + + with open(f'ablation_results_{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.json', 'w') as outfile: + json.dump(results, outfile) + + From 89b82022c1847f4679ec6d15b1f4552e571e8c79 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 7 Aug 2026 16:32:59 -0600 Subject: [PATCH 68/81] Added logger to app and changed print statements in except statements to use logger.exception --- app/__init__.py | 3 +-- app/db_utils/data_profile.py | 32 +++++++++++++++----------------- app/server_utils/logger_utils.py | 14 +++++++------- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 980f3e0..e09959b 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -30,6 +30,7 @@ def format(self, record): _werkzeug_handler = logging.StreamHandler() _werkzeug_handler.setFormatter(_WhiteFormatter('%(message)s')) logging.getLogger('werkzeug').handlers = [_werkzeug_handler] +logger = logging.getLogger('werkzeug') from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT @@ -107,8 +108,6 @@ def load_database_info(): # Create the database if it does not exist create_database_if_not_exists(connection, db_name) - - """ then we use SQLAlchemy (create_engine) for everything else this is the engine that gets passed around to fetch_sql, execute_sql, diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index b360087..34b137e 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -3,6 +3,7 @@ import json from pandas.core.arrays import categorical +from app import db_operations, logger from app.db_utils.execute_sql import fetch_sql from app.db_utils.column_types import ColumnTypes @@ -83,11 +84,8 @@ def get_col_names(self): if row[0] not in ['index', 'level_0', ]: col_names.append(row[0]) - print("COL NAMES FROM QUERY: ", col_names) - - except Exception as e: - print(f"AHHHHHHHHHH Querying for col names unsuccessful because of error: {e}") + logger.exception("Error when querying for col names") return col_names @@ -105,7 +103,7 @@ def look_up_stat_from_profile(self, attribute_name, column_name): return stat except Exception as e: - print(f"Error querying attribute from data profile table: {e}") + logger.exception(f"Error querying attribute from data profile table") return None @@ -160,7 +158,7 @@ def _calculate_mean(self, column_name): try: avg = self.calculate_summary_stat_using_sql('AVG', column_name) except Exception as e: - print(f"Error fetching the mean for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the mean for table {self.table_name} at column {column_name}") avg = None @@ -183,7 +181,7 @@ def _calculate_median(self, column_name): query += f' WHERE pg_input_is_valid("{column_name}", \'numeric\')' median = fetch_sql(query, True, self.engine) except Exception as e: - print(f"Error fetching the median for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the median for table {self.table_name} at column {column_name}") median = float('nan') return median @@ -199,7 +197,7 @@ def _calculate_max(self, column_name): try: maximum = self.calculate_summary_stat_using_sql('MAX', column_name) except Exception as e: - print(f"Error fetching the maximum for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the maximum for table {self.table_name} at column {column_name}") maximum = None @@ -215,7 +213,7 @@ def _calculate_min(self, column_name): try: minimum = self.calculate_summary_stat_using_sql('MIN', column_name) except Exception as e: - print(f"Error fetching the minimum for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the minimum for table {self.table_name} at column {column_name}") minimum = None @@ -233,8 +231,8 @@ def _calculate_num_categories(self, column_name): n_categories = fetch_sql(query, True, self.engine) except Exception as e: - print("AHHH SQL QUERY DIDN'T WORK") - print(f"Error fetching the n_categories for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the n_categories for table {self.table_name} at column {column_name}") + n_categories = None @@ -259,8 +257,7 @@ def _calculate_mode(self, column_name): print("MODE FROM SQL QUERY: ", mode) except Exception as e: - print("AHHH SQL QUERY DIDN'T WORK") - print(f"Error fetching the mode for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the mode for table {self.table_name} at column {column_name}") mode = None @@ -288,7 +285,7 @@ def _calculate_error_count_dict(self, column_name): error_counts = json.dumps({}) except Exception as e: - print(f"Error fetching the error counts for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the error counts for table {self.table_name} at column {column_name}") error_counts = None @@ -365,8 +362,7 @@ def _calculate_category_count_dict(self, column_name): except Exception as e: - - print(f"Error fetching the category counts for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the category counts for table {self.table_name} at column {column_name}") category_counts = None if category_counts is not None: @@ -400,4 +396,6 @@ def _calculate_class_error_count_dict(self, column_name): counts_by_column = json.dumps(counts_by_column) - return counts_by_column + return counts_by_column + except Exception as e: + logger.exception(f"Error fetching the class error counts for table {self.table_name}") diff --git a/app/server_utils/logger_utils.py b/app/server_utils/logger_utils.py index a98074c..d29c74b 100644 --- a/app/server_utils/logger_utils.py +++ b/app/server_utils/logger_utils.py @@ -6,12 +6,12 @@ from datetime import datetime, timezone from sqlalchemy import Text, TIMESTAMP, Boolean, Float from sqlalchemy.dialects.postgresql import UUID # if using Postgres -import logging +from app import logger import json from app.db_utils.execute_sql import fetch_sql -logger = logging.getLogger(__name__) + ACTION_LOG_TABLE_NAME = "action_log" PREVIEW_LOG_TABLE_NAME = "preview_log" @@ -45,7 +45,7 @@ def update_action_log(dataset_id, action_name, action_details, engine, timestamp new_action_entry.to_sql(ACTION_LOG_TABLE_NAME, engine, if_exists='append', index=False) print("UPDATED ACTION LOG TABLE") except Exception: - logger.error("Error updating action log.", exc_info=True) + logger.exception("Error updating action log.") @@ -71,7 +71,7 @@ def initialize_action_log(engine, reset_log=False): else: empty_log_df.to_sql(ACTION_LOG_TABLE_NAME, engine, if_exists='append', index=False, dtype=dtype_map) except Exception: - logger.error("Error initializing action log.", exc_info=True) + logger.exception("Error initializing action log.") def initialize_preview_log_table(engine, reset_log=False): print("INITIALIZING PREVIEW LOG TABLE") @@ -90,7 +90,7 @@ def initialize_preview_log_table(engine, reset_log=False): empty_log_df.to_sql(PREVIEW_LOG_TABLE_NAME, engine, if_exists='append', index=False, dtype=dtype_map) print("INITIALIZED PREVIEW LOG TABLE") except Exception: - logger.error("Error initializing preview log table.", exc_info=True) + logger.exception("Error initializing preview log table.") def update_preview_log(preview_table_name, action_name, action_details, engine): @@ -103,7 +103,7 @@ def update_preview_log(preview_table_name, action_name, action_details, engine): new_action_entry.to_sql(PREVIEW_LOG_TABLE_NAME, engine, if_exists='append', index=False) print("UPDATED PREVIEW LOG TABLE") except Exception: - logger.error("Error updating preview log table.", exc_info=True) + logger.exception("Error updating preview log table.") def get_action_details_from_preview_log(preview_table_name, engine): try: @@ -117,7 +117,7 @@ def get_action_details_from_preview_log(preview_table_name, engine): result = fetch_sql(query, params={"id": preview_table_name},scalar=True, engine=engine) return result except Exception: - logger.error(f"Error retrieving action details from {preview_table_name} from preview log.", exc_info=True) + logger.exception(f"Error retrieving action details from {preview_table_name} from preview log.") result = None return result From 53fbac817ff5da0972e1b5f94d639541c0379387 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Fri, 7 Aug 2026 16:38:22 -0600 Subject: [PATCH 69/81] Implemented copy_table_to_csv function in execute_sql.py --- app/db_utils/execute_sql.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/app/db_utils/execute_sql.py b/app/db_utils/execute_sql.py index f80cf96..d3cc1bc 100644 --- a/app/db_utils/execute_sql.py +++ b/app/db_utils/execute_sql.py @@ -33,3 +33,17 @@ def fetch_sql(query: str, scalar: bool, engine, params=None): return None +def copy_table_to_csv(table_name: str, csv_file_path: str, engine): + """ + Copies the contents of a PostgreSQL table to a CSV file. + :arg: table_name: name of the table to copy. + :arg: csv_file_path: path to the CSV file to write to. + """ + print("table_name", table_name) + print("csv_file_path", csv_file_path) + query = f'COPY "{table_name}" TO STDOUT WITH CSV HEADER' + with engine.raw_connection() as conn: + with open(csv_file_path, 'w') as f: + cursor = conn.cursor() + cursor.copy_expert(query, f) + cursor.close() \ No newline at end of file From 2a45fe8dbcddfefb8e0548db98268cfb15eb2a6d Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 10 Aug 2026 14:30:08 -0600 Subject: [PATCH 70/81] Fixed logic bug relating to calculating column stats in service_helpers.py --- app/server_utils/service_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/server_utils/service_helpers.py b/app/server_utils/service_helpers.py index f97937f..b55d5f5 100644 --- a/app/server_utils/service_helpers.py +++ b/app/server_utils/service_helpers.py @@ -211,8 +211,8 @@ def create_data_profile_df(data_profile, col_names=None): # Make sure that attribute and the column type match - numeric = ((data_profile.col_types.is_numeric_mixed_col(col) or data_profile.col_types.is_numeric_col )and attribute in data_profile.attribute_type_assignment['numeric']) - categorical = ((data_profile.col_types.is_categorical_mixed_col(col) or data_profile.col_types.is_categorical_col)and attribute in data_profile.attribute_type_assignment['categorical']) + numeric = ((data_profile.col_types.is_numeric_mixed_col(col) or data_profile.col_types.is_numeric_col(col) )and attribute in data_profile.attribute_type_assignment['numeric']) + categorical = ((data_profile.col_types.is_categorical_mixed_col(col) or data_profile.col_types.is_categorical_col(col))and attribute in data_profile.attribute_type_assignment['categorical']) if not (numeric or categorical): From 9f98892546a119fc6957285336c8851d7253bcfd Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 10 Aug 2026 14:31:21 -0600 Subject: [PATCH 71/81] Added params argument to execute_sql function --- app/db_utils/execute_sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/db_utils/execute_sql.py b/app/db_utils/execute_sql.py index d3cc1bc..b980727 100644 --- a/app/db_utils/execute_sql.py +++ b/app/db_utils/execute_sql.py @@ -4,13 +4,13 @@ Thin wrapper around SQLAlchemy for executing and fetching results from a PostgreSQL database. """ -def execute_sql(query: str, engine): +def execute_sql(query: str, engine, params=None): """ Executes given SQL query to the postgres database. """ with engine.begin() as conn: - conn.execute(text(query)) + conn.execute(text(query), params) def fetch_sql(query: str, scalar: bool, engine, params=None): """ From 93d120e6389a763c13a08d084d80db5c95610e51 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 10 Aug 2026 14:31:55 -0600 Subject: [PATCH 72/81] Implemented create_empty_settings_df function --- app/server_utils/logger_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/server_utils/logger_utils.py b/app/server_utils/logger_utils.py index d29c74b..4c02083 100644 --- a/app/server_utils/logger_utils.py +++ b/app/server_utils/logger_utils.py @@ -25,6 +25,9 @@ def create_empty_preview_log_df(): empty_df = pd.DataFrame(columns=['preview_table_name', 'action_name', 'action_details']) return empty_df +def create_empty_settings_df(): + empty_df = pd.DataFrame(columns=['model']) + return empty_df # TODO: update documentation def update_action_log(dataset_id, action_name, action_details, engine, timestamp, action_successful, From 73c3c588e1fed215e17dd10f679a9374d3517ea4 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 10 Aug 2026 14:36:56 -0600 Subject: [PATCH 73/81] Fixed action_duration related bugs & cleaned up code --- app/routes/routes.py | 2 +- app/routes/wrangler_routes_sql.py | 23 ++++++++--------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/app/routes/routes.py b/app/routes/routes.py index 99c8546..b96d563 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -86,7 +86,7 @@ def load_file(csv_file, filename): #init the pgraph init_pgraph_for_session(table_name_with_node_id) - action_duration = datetime.now(timezone.utc) - timestamp + action_duration = (datetime.now(timezone.utc) - timestamp).total_seconds() update_action_log(dataset_id=base_table_name, action_name="load_dataset", action_details=None, engine=engine, timestamp=timestamp, action_duration=action_duration, action_successful=True) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index af5f962..25ac39d 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -225,13 +225,12 @@ def create_previews(): "dims": 2, } - action_duration = datetime.now(timezone.utc) - timestamp - action_duration_seconds = action_duration.total_seconds() + action_duration = (datetime.now(timezone.utc) - timestamp).total_seconds() update_action_log(dataset_id=table, action_name="create_previews", action_details=json.dumps(action_details_dict), engine=engine, - timestamp=timestamp, action_duration= action_duration_seconds,action_successful=True) + timestamp=timestamp, action_duration= action_duration,action_successful=True) assert result_dict is not None @@ -262,17 +261,12 @@ def execute_wrangle(): body = request.get_json(force=True) table = db_operations.main_table_name preview_table = body["preview_table"] # the preview to promote - action_details_dict = get_action_details_from_preview_log(preview_table, engine) - - wrangle_executed = extract_preview_action(preview_table) - - new_table_name = execute_wrangle_preview(table, preview_table, _safe_pg_name, db_operations) - # TODO: incorporate action details from preview log table into this - action_duration = datetime.now(timezone.utc) - timestamp - action_duration_seconds = action_duration.total_seconds() + (new_table_name, action_details_dict, wrangle_executed) = execute_wrangle_logic(preview_table, table) + + action_duration = (datetime.now(timezone.utc) - timestamp).total_seconds() update_action_log(dataset_id=db_operations.base_table_name, action_name=f"{wrangle_executed}_wrangle", - action_details=action_details_dict, engine=db_operations.engine, timestamp=timestamp, action_duration= action_duration_seconds,action_successful=True) + action_details=action_details_dict, engine=db_operations.engine, timestamp=timestamp, action_duration= action_duration,action_successful=True) return {"success": True, "table": new_table_name} except Exception as e: @@ -308,10 +302,9 @@ def wrangle_delete_column(): # Re-run error detection update_errors_table(table_name, [column]) update_data_profile_table(table_name, [column]) - action_duration = datetime.now(timezone.utc) - timestamp - action_duration_seconds = action_duration.total_seconds() + action_duration = (datetime.now(timezone.utc) - timestamp).total_seconds() update_action_log(dataset_id=db_operations.base_table_name, action_name="delete_column", - action_details={"column": column}, engine=engine, timestamp=timestamp, action_duration= action_duration_seconds, + action_details={"column": column}, engine=engine, timestamp=timestamp, action_duration= action_duration, action_successful=True) From 02e944c24a06ea84b6b212e77c9d091459ebe153 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 10 Aug 2026 14:37:33 -0600 Subject: [PATCH 74/81] Fixed if statement in data_profile.py --- app/db_utils/data_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index 34b137e..0520c21 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -135,7 +135,7 @@ def calculate_column_attribute(self, attribute_name, column_name, look_up_stat=T :param look_up_stat: If True, first attempt to look up the statistic from the data profile table. If False, calculate it directly. :return: The value of the statistic if found or calculated, otherwise None. """ - if look_up_stat: + if look_up_stat and db_operations.table_exists(self.data_profile_table_name): look_up_value = self.look_up_stat_from_profile(attribute_name, column_name) if look_up_value is not None: From 89d41b16d0c76ce64caffd048955973c3285bd0f Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 10 Aug 2026 14:38:35 -0600 Subject: [PATCH 75/81] Added try except block in data_profile.py (_calculate_class_error_count_dict) --- app/db_utils/data_profile.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index 0520c21..4a8c190 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -381,19 +381,18 @@ def _calculate_class_error_count_dict(self, column_name): # TODO: Implement SQL query version print("Calculating class error counts manually using data...") - self.load_error_df() - - counts_by_column = {} - if not self._error_df.empty: # If error_df is empty (no errors in data selection) - counts_by_column = ( - self._error_df.groupby(['column_id', 'error_type']) - .size() - .unstack(fill_value=0) - .to_dict(orient='index') - ) - - if counts_by_column is not None: - counts_by_column = json.dumps(counts_by_column) + try: + counts_by_column = {} + if not self._error_df.empty: # If error_df is empty (no errors in data selection) + counts_by_column = ( + self._error_df.groupby(['column_id', 'error_type']) + .size() + .unstack(fill_value=0) + .to_dict(orient='index') + ) + + if counts_by_column is not None: + counts_by_column = json.dumps(counts_by_column) return counts_by_column From a018c3c2b15967d564abef01a8d576e39b6d4021 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 10 Aug 2026 14:39:34 -0600 Subject: [PATCH 76/81] Created ai_routes.py --- app/routes/ai_routes.py | 306 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 app/routes/ai_routes.py diff --git a/app/routes/ai_routes.py b/app/routes/ai_routes.py new file mode 100644 index 0000000..9542cab --- /dev/null +++ b/app/routes/ai_routes.py @@ -0,0 +1,306 @@ + +from litellm import completion, get_max_tokens, token_counter +from app import app, engine, logger +from app.db_utils.ai_utils import update_csvs_for_llm, get_api_key, parse_json_response +from app.db_utils.execute_sql import fetch_sql, execute_sql + +from app.db_utils.ai_utils import call_with_retry + +AI_SETTINGS_TABLE_NAME = "ai_settings" +from flask import request + +ablations = ["include_data_profile", "include_dataset_context", "include_action_plan", "include_action_plan_translation"] +valid_actions = ["delete_wrangle", "impute_wrangle", "delete_column", "plan_end"] + +''' + LLM Query plans: + - just the action plan in json + - action plan in text -> translate to json + - +''' + + + +# TODO: either put this back to action_details or apply rows and columns to the rest of the logs + +def query_llm_for_text_action_plan(model, provider, api_key, error_log_csv_path, action_log_csv_path, + data_profile_csv_path, full_dataset_csv_path, action_limit=5): + system_prompt = f"You are data scientist. Create an action plan of the top {action_limit} steps the user could do that efficiently cleans this dataset." + + csv_text = {} + + with open(action_log_csv_path, "r") as f: + csv_text[action_log_csv_path] = f.read() + + with open(data_profile_csv_path, "r") as f: + csv_text[data_profile_csv_path] = f.read() + + with open(error_log_csv_path, "r") as f: + csv_text[error_log_csv_path] = f.read() + + with open(full_dataset_csv_path, "r") as f: + csv_text[full_dataset_csv_path] = f.read() + + llm_text_plan_rules = ''' + Each step of the action plan should be numbered and begin with the selection of columns and rows + to perform the wrangles on. You may also select the whole dataset by putting "ALL" for "rows" and "columns". At each step, determine a SINGULAR wrangling + action to perform on the selected data. Here are the wrangles that you can perform on the data: + - Delete: deletes the selection of data + - Impute: impute the selection of data with either the mean if numeric or the mode if categorical + - Delete column: deletes a column of the dataset + - Stop: stop wrangling the data as the data is at a satisfactory state. This does not have to be the last action in the plan. + The stop action should only be used if there are no more actions needed to be done. + + OTHER RULES THAT YOU MUST FOLLOW + - DO NOT CREATE NEW ACTIONS + - ONLY OUTPUT THE ACTION PLAN, NOTHING ELSE + ''' + + additional_details = f''' Here are the contents of each of the CSVs that you can use to inform the actions in your action plan: + action_log: {csv_text[action_log_csv_path]}\n + data_profile: {csv_text[data_profile_csv_path]}\n + error_log: {csv_text[error_log_csv_path]}\n + full_dataset: {csv_text[full_dataset_csv_path]}\n + ''' + + full_message = llm_text_plan_rules + additional_details + print("full_message", full_message) + print("len(full_message)", len(full_message)) + print("estimated num tokens", len(full_message) / 4) + full_model_name = provider + '/' + model + print("MAX TOKENS", get_max_tokens(full_model_name)) + print("TOKEN COUNT", token_counter(model=full_model_name, text=full_message)) + + response = completion( + model=full_model_name, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": full_message}, + ], + api_key=api_key + ) + + llm_text_response = response.choices[0].message.content + print("LLM TEXT RESPONSE", llm_text_response) + + return llm_text_response + + +def query_llm_for_action_plan_translation(model, provider, api_key, text_action_plan): + system_prompt = "You are a translator that translates actions from natural language text to JSON where each row is an action. Translate this action plan from text to JSON." + + llm_translated_plan_rules = ''' + Each step of the action plan should be a dict with the following keys: "action_name", "rows_to_wrangle", "column". + Here are the valid actions and their names: + - Delete -> "delete_wrangle" + - Impute -> "impute_wrangle" + - Delete column -> "delete_column" + - Stop -> "plan_end" + + The value of "rows_to_wrangle" is list of rows to wrangle. + The value of "column" is the name of a SINGLE column to wrangle. + + OTHER RULES THAT YOU MUST FOLLOW + - DO NOT CREATE NEW ACTIONS + - THE RESULT MUST BE IN JSON FORMAT + ''' + + text_action_plan_prompt = f''' + Here is the text action plan you must translate to JSON: + {text_action_plan} + ''' + + response = completion( + model=provider + '/' + model, + response_format={"type": "json_object"}, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": llm_translated_plan_rules + text_action_plan_prompt}, + ], + api_key=api_key + ) + + llm_text_response = response.choices[0].message.content + print("LLM TEXT TRANSLATION RESPONSE", llm_text_response) + + json_action_plan = parse_json_response(llm_text_response) + + return json_action_plan + + +def query_llm_for_dataset_context(model, api_key, column_names, user_provided_dataset_context, dataset_name): + system_prompt = ("You are a data scientist that is given a dataset. " + "You are to provide context about the dataset. You are to give a description of the dataset as well" + "as a brief description of each column. If you do not know what a column is, you can have the" + "description be 'N/A'") + + llm_dataset_context_rules = f''' + The dataset is named: {dataset_name} + The columns in the dataset are: {column_names} + The user has provided the following context about the dataset: {user_provided_dataset_context} + + Using this information, create a JSON object with the following keys: 'dataset_description', 'column_descriptions'. + The value of 'dataset_description' is the description of the dataset. + The value of 'column_descriptions' is a dictionary where the keys are the column names and the values are the descriptions of each column. + If you do not know what a column is, you can have the description be None. + ''' + + response = completion( + model=model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": llm_dataset_context_rules}, + ], + api_key=api_key + ) + + llm_text_response = response.choices[0].message.content + return llm_text_response + +@app.post('/api/ai_helper/perform_llm_action') +def perform_llm_action(): + from app import db_operations + from app.db_utils.query import remove_rows_by_ids, impute_by_ids + action_dict = request.get_json() + + try: + main_table_name = db_operations.main_table_name + action_name = action_dict["action_name"] + rows_to_wrangle = action_dict["rows_to_wrangle"] + column = action_dict["column"] + + if action_name == "stop": + return { + "success": True + } + + # Apply the changes directly to the table (no preview) + if action_name == "delete": + remove_rows_by_ids(table=main_table_name, ids=rows_to_wrangle) + elif action_name == "impute": + impute_by_ids(table=main_table_name, col=column,ids=rows_to_wrangle) + + return { + "success": True + } + + + except Exception as e: + logger.exception("Error performing LLM action") + return { + "success": False + } + +@app.post('/api/ai_helper/get_action_plan') +def get_llm_json_action_plan(): + from app import db_operations, engine + from app.server_utils.logger_utils import ACTION_LOG_TABLE_NAME + try: + + error_table_name = db_operations.error_table_name + data_profile_name = db_operations.dp_table_name + full_dataset_name = db_operations.main_table_name + action_log_name = ACTION_LOG_TABLE_NAME + settings_dict = get_settings_dict(engine) + model_name = settings_dict.get("model_name") + provider = settings_dict.get("provider") + api_key = get_api_key(provider) + + assert model_name is not None + + (error_log_csv_path, data_profile_csv_path, action_log_csv_path, full_dataset_csv_path) = update_csvs_for_llm(error_table_name, + data_profile_name, + action_log_name, full_dataset_name) + + text_plan_func_args = (model_name, provider, api_key, action_log_csv_path, + error_log_csv_path, data_profile_csv_path, full_dataset_csv_path) + text_action_plan = call_with_retry(query_llm_for_text_action_plan, text_plan_func_args, max_tries=5) + + translation_func_args = (model_name, provider, api_key, text_action_plan) + + json_action_plan = call_with_retry(query_llm_for_action_plan_translation, translation_func_args, + max_tries=5) + + return {"success": True, "json_action_plan": json_action_plan} + except Exception as e: + json_action_plan = None + logger.exception("Error translating llm action plan to json") + return {"success": False, "json_action_plan": json_action_plan} + + +@app.post('/api/ai_helper/update_settings_table') +def update_settings_table(): + print("UPDATING SETTINGS TABLE") + data = request.get_json() + model_name = data.get("model_name") + provider = data.get("provider") + print("MODEL NAME", model_name, "PROVIDER", provider) + + try: + execute_sql(f""" + CREATE TABLE IF NOT EXISTS {AI_SETTINGS_TABLE_NAME} + ( + id + INTEGER + PRIMARY + KEY, + model_name + TEXT + NOT + NULL, + provider + TEXT + NOT + NULL + ) + """, engine) + + result = fetch_sql(f"SELECT id FROM {AI_SETTINGS_TABLE_NAME} WHERE id = :id", True, + engine, {"id": 1} + ) + + if result is None: + print("VALUE DOESN'T EXIST. INSERTING ONE") + execute_sql( + f"INSERT INTO {AI_SETTINGS_TABLE_NAME} (id, model_name, provider) VALUES (:id, :model_name, :provider)", + engine, {"id": 1, "model_name": model_name, "provider": provider} + ) + else: + print("VALUE EXISTS, REPLACING") + execute_sql( + f"UPDATE {AI_SETTINGS_TABLE_NAME} SET model_name = :model_name, provider = :provider WHERE id = :id", + engine, {"id": 1, "model_name": model_name, "provider": provider} + ) + + print("MODEL NAME", model_name, "PROVIDER", provider) + return {"success": True} + except Exception as e: + logger.exception("Error updating settings table.") + + return {"success": False} + +# TODO: what do i do if get_settings_dict is None? +def get_settings_dict(engine): + try: + result = fetch_sql(f"SELECT model_name, provider FROM {AI_SETTINGS_TABLE_NAME} WHERE id = :id", False, engine, {"id": 1}) + row = result[0] + + if result is not None: + model_name = row.model_name # or row[0] + provider = row.provider + print("MODEL NAME", model_name, "PROVIDER", provider) + # TODO: maybe don't return a dict + settings_dict = {"model_name": model_name, "provider": provider} + else: + settings_dict = None + + return settings_dict + except Exception as e: + logger.exception("Error retrieving settings from table.") + raise + + + + + + From c203271259f6bb4b53a11dbb14734e2aa5e183c9 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 10 Aug 2026 14:39:53 -0600 Subject: [PATCH 77/81] Created ai_utils.py --- app/db_utils/ai_utils.py | 97 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 app/db_utils/ai_utils.py diff --git a/app/db_utils/ai_utils.py b/app/db_utils/ai_utils.py new file mode 100644 index 0000000..cf412c8 --- /dev/null +++ b/app/db_utils/ai_utils.py @@ -0,0 +1,97 @@ +import os +from app.db_utils.execute_sql import copy_table_to_csv +from app import logger +import json +import ast + +import random +import time + + +# Overwriting the csv every time the LLM needs it to be updated; we don't really need to save the old ones + +def update_csvs_for_llm(error_table_name, data_profile_name, action_log_name, full_dataset_name): + print(f"UPDATE CSVS FOR LLM PATHS error_table_name: {error_table_name}, data_profile_name: {data_profile_name} action_log_name: {action_log_name}") + + action_log_csv_path = "action_log.csv" + error_log_csv_path = "error_log.csv" + data_profile_csv_path = "data_profile.csv" + full_dataset_csv_path = "full_dataset.csv" + + _THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + FILES_FOR_LLM_PATH = os.path.abspath(os.path.join(_THIS_DIR, '..', 'files_for_llm')) + + action_log_csv_path = FILES_FOR_LLM_PATH + '/' + f'{action_log_csv_path}' + error_log_csv_path = FILES_FOR_LLM_PATH + '/' + f'{error_log_csv_path}' + data_profile_csv_path = FILES_FOR_LLM_PATH + '/' + f'{data_profile_csv_path}' + full_dataset_csv_path = FILES_FOR_LLM_PATH + '/' + f'{full_dataset_csv_path}' + + + table_name_tuple_list = [(action_log_name, action_log_csv_path), + (error_table_name, error_log_csv_path), + (data_profile_name, data_profile_csv_path), + (full_dataset_name, full_dataset_csv_path)] + + write_tables_to_csv(table_name_tuple_list) + + return (error_log_csv_path, data_profile_csv_path, action_log_csv_path, full_dataset_csv_path) + + + +def write_tables_to_csv(table_name_tuple_list): + from app import engine + for (table_name, csv_path) in table_name_tuple_list: + + # Clear the existing CSV file if it exists + if os.path.exists(csv_path): + os.remove(csv_path) + + copy_table_to_csv(table_name, csv_path, engine) + + +def parse_json_response(llm_json_response): + try: + return json.loads(llm_json_response) + except json.JSONDecodeError: + try: + return ast.literal_eval(llm_json_response) + except (ValueError, SyntaxError) as e: + logger.exception(f"Could not parse response as dict or JSON") + + +def call_with_retry(function, func_args, max_tries=5): + for attempt in range(max_tries): + try: + result = function(*func_args) + + return result + except Exception: + logger.exception("Error occurred while calling LLM function") + if attempt == max_tries - 1: + raise + + # TODO: is this okay + delay = 10 + delay *= random.uniform(0.5, 1.5) # jitter + time.sleep(delay) + + + +def get_api_key(provider): + key_map = { + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "groq": "GROQ_API_KEY" + } + + key = os.environ.get(key_map[provider]) + if key is None: + raise ValueError(f"Could not find API key for {provider}") + return key + + + + + + + From 7fc83ab7f2b7ce8d55d765cbbbe557d7be9d8f3a Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 10 Aug 2026 14:40:58 -0600 Subject: [PATCH 78/81] Separated execute_wrangle logic from execute_wrangle api call --- app/routes/wrangler_routes_sql.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index 25ac39d..ca9f6df 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -148,6 +148,17 @@ def update_preview_error_table(table_name: str, err_table_name: str) -> None: traceback.print_exc() raise +def execute_wrangle_logic(preview_table, table): + action_details_dict = get_action_details_from_preview_log(preview_table, engine) + + wrangle_executed = extract_preview_action(preview_table) + + new_table_name = execute_wrangle_preview(table, preview_table, _safe_pg_name, db_operations) + + return (new_table_name, action_details_dict, wrangle_executed) + + + # ───────────────────────────────────────────────────────────────────────────── # Wrangling Endpoints (Supports both bin-based and ID-based selections) # the way it works is, create-previews does all wrangles (delete, impute x/y), From 23f956363902bd5a6cd4d860fac3f45a7237fbff Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 10 Aug 2026 14:41:41 -0600 Subject: [PATCH 79/81] Created __init__.py for ablation study directory --- app/ablation_study/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 app/ablation_study/__init__.py diff --git a/app/ablation_study/__init__.py b/app/ablation_study/__init__.py new file mode 100644 index 0000000..e69de29 From 7a29779c685ff6b6759ffd800d37450d78f7fea1 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 10 Aug 2026 14:44:52 -0600 Subject: [PATCH 80/81] Fixed ablation study config dict --- app/ablation_study/ablation_study.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/ablation_study/ablation_study.py b/app/ablation_study/ablation_study.py index fd9d21d..84cac0f 100644 --- a/app/ablation_study/ablation_study.py +++ b/app/ablation_study/ablation_study.py @@ -47,9 +47,9 @@ def is_stop_action(action_name): ablation_configs = [ variant("baseline"), variant("no_error_log", include_error_log=False), - variant("no_data_profile", include_error_log=False), - variant("no_action_log", include_error_log=False), - variant("no_full_dataset", include_error_log=False), + variant("no_data_profile", include_data_profile=False), + variant("no_action_log", include_action_log=False), + variant("no_full_dataset", include_full_dataset=False), variant("no_action_log_limit", action_log_limit=None) # Includes full action log ] @@ -77,6 +77,7 @@ def is_stop_action(action_name): # reset your stateful class instance db_operations.reset() for dataset in datasets_paths: + print(f"Running config {config} wth model {model} and dataset {dataset}") initialize_action_log(engine, reset_log=True) with open(dataset, 'rb') as f: From 9d7c6c8bb1fa50fe12d64e311c784a2d40bdc3c0 Mon Sep 17 00:00:00 2001 From: marikmartinez Date: Mon, 10 Aug 2026 15:09:17 -0600 Subject: [PATCH 81/81] Added todo for later --- app/server_utils/logger_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server_utils/logger_utils.py b/app/server_utils/logger_utils.py index 4c02083..3c319ac 100644 --- a/app/server_utils/logger_utils.py +++ b/app/server_utils/logger_utils.py @@ -16,7 +16,7 @@ ACTION_LOG_TABLE_NAME = "action_log" PREVIEW_LOG_TABLE_NAME = "preview_log" -# TODO: switch all of the sql table creation functions to not use pandas because apparently it's very slow +# TODO: make it so initialization of log is combined with updating the log def create_empty_user_action_log_df(): empty_df = pd.DataFrame(columns=['action_id', 'dataset_id', 'action_name', 'action_details', 'timestamp', 'action_duration', 'action_successful', 'action_error_message']) return empty_df