diff --git a/app/__init__.py b/app/__init__.py index 980f3e0..e09959b 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -30,6 +30,7 @@ def format(self, record): _werkzeug_handler = logging.StreamHandler() _werkzeug_handler.setFormatter(_WhiteFormatter('%(message)s')) logging.getLogger('werkzeug').handlers = [_werkzeug_handler] +logger = logging.getLogger('werkzeug') from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT @@ -107,8 +108,6 @@ def load_database_info(): # Create the database if it does not exist create_database_if_not_exists(connection, db_name) - - """ then we use SQLAlchemy (create_engine) for everything else this is the engine that gets passed around to fetch_sql, execute_sql, diff --git a/app/ablation_study/__init__.py b/app/ablation_study/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/ablation_study/ablation_study.py b/app/ablation_study/ablation_study.py new file mode 100644 index 0000000..84cac0f --- /dev/null +++ b/app/ablation_study/ablation_study.py @@ -0,0 +1,127 @@ +import os + +from dotenv import load_dotenv +import datetime + + +from app import app as app_module +from app import db_operations, engine + +import json + +from app.db_utils.ai_utils import parse_json_response +from app.server_utils.logger_utils import initialize_action_log + +datasets_paths = ['provided_datasets/mari_dataset.csv'] +#models = [{"model": "qwen/qwen3.6-27b", "provider": "groq"}] +models = [{"model": "openai/gpt-oss-20b", "provider": "groq"}] + +def variant(name, **overrides): + baseline = { + "name": "baseline", + "include_error_log": True, + "include_data_profile": True, + + "include_action_log": True, + "action_log_limit": 10, + "include_full_dataset": True + } + + # gets the baseline and overrides the key(s) in the overrides variable + config = {**baseline, **overrides} + config["name"] = name + return config + + +def is_stop_action(action_name): + if action_name == "stop": + return True + else: + return False + + +if __name__ == "__main__": + load_dotenv() + + # TODO: double check this + ablation_configs = [ + variant("baseline"), + variant("no_error_log", include_error_log=False), + variant("no_data_profile", include_data_profile=False), + variant("no_action_log", include_action_log=False), + variant("no_full_dataset", include_full_dataset=False), + variant("no_action_log_limit", action_log_limit=None) # Includes full action log + ] + + client = app_module.test_client() + app_module.testing = False + + results = [] + + for model_dict in models: + + model = model_dict["model"] + model_provider = model_dict["provider"] + + update_settings_table_result = client.post('/api/ai_helper/update_settings_table', json={"model_name": model, "provider": model_provider}) + data = update_settings_table_result.get_json() + assert data["success"] == True + + for config in ablation_configs: + # reset globals living in the app package namespace + app_module.wrangle_occurred = False + + # reset attributes on the Flask object itself + app_module.pgraph_for_session = None + + # reset your stateful class instance + db_operations.reset() + for dataset in datasets_paths: + print(f"Running config {config} wth model {model} and dataset {dataset}") + initialize_action_log(engine, reset_log=True) + + with open(dataset, 'rb') as f: + upload_result = client.post('/api/upload', data={'file': (f, dataset)}, + content_type='multipart/form-data') + data = upload_result.get_json() + assert data["success"] == True + + result_dict = {} # Dict added to the result json + result_dict["model"] = model_dict + result_dict["config_name"] = config["name"] + result_dict["dataset"] = dataset + result_dict["actions"] = [] + + action_plan_batch = 0 + + stop_action_found = False + + while not stop_action_found: + action_plan_batch += 1 + + actions = None + + # Get actions + get_action_plan_result = client.post('/api/ai_helper/get_action_plan') + action_plan_result_json = get_action_plan_result.get_json() + assert action_plan_result_json["success"] == True + action_plan_json = action_plan_result_json["json_action_plan"] + + # Go through all actions and perform each of them + for action_dict in action_plan_json: + + if is_stop_action(action_dict["action_name"]): + stop_action_found = True + + action_result = client.post('/api/ai_helper/perform_llm_action', json=action_dict) + action_dict["action_plan_batch"] = action_plan_batch + action_dict["timestamp"] = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + action_dict["success"] = action_result.get_json() + + results.append(action_dict) + + + with open(f'ablation_results_{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.json', 'w') as outfile: + json.dump(results, outfile) + + diff --git a/app/db_utils/ai_utils.py b/app/db_utils/ai_utils.py new file mode 100644 index 0000000..cf412c8 --- /dev/null +++ b/app/db_utils/ai_utils.py @@ -0,0 +1,97 @@ +import os +from app.db_utils.execute_sql import copy_table_to_csv +from app import logger +import json +import ast + +import random +import time + + +# Overwriting the csv every time the LLM needs it to be updated; we don't really need to save the old ones + +def update_csvs_for_llm(error_table_name, data_profile_name, action_log_name, full_dataset_name): + print(f"UPDATE CSVS FOR LLM PATHS error_table_name: {error_table_name}, data_profile_name: {data_profile_name} action_log_name: {action_log_name}") + + action_log_csv_path = "action_log.csv" + error_log_csv_path = "error_log.csv" + data_profile_csv_path = "data_profile.csv" + full_dataset_csv_path = "full_dataset.csv" + + _THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + FILES_FOR_LLM_PATH = os.path.abspath(os.path.join(_THIS_DIR, '..', 'files_for_llm')) + + action_log_csv_path = FILES_FOR_LLM_PATH + '/' + f'{action_log_csv_path}' + error_log_csv_path = FILES_FOR_LLM_PATH + '/' + f'{error_log_csv_path}' + data_profile_csv_path = FILES_FOR_LLM_PATH + '/' + f'{data_profile_csv_path}' + full_dataset_csv_path = FILES_FOR_LLM_PATH + '/' + f'{full_dataset_csv_path}' + + + table_name_tuple_list = [(action_log_name, action_log_csv_path), + (error_table_name, error_log_csv_path), + (data_profile_name, data_profile_csv_path), + (full_dataset_name, full_dataset_csv_path)] + + write_tables_to_csv(table_name_tuple_list) + + return (error_log_csv_path, data_profile_csv_path, action_log_csv_path, full_dataset_csv_path) + + + +def write_tables_to_csv(table_name_tuple_list): + from app import engine + for (table_name, csv_path) in table_name_tuple_list: + + # Clear the existing CSV file if it exists + if os.path.exists(csv_path): + os.remove(csv_path) + + copy_table_to_csv(table_name, csv_path, engine) + + +def parse_json_response(llm_json_response): + try: + return json.loads(llm_json_response) + except json.JSONDecodeError: + try: + return ast.literal_eval(llm_json_response) + except (ValueError, SyntaxError) as e: + logger.exception(f"Could not parse response as dict or JSON") + + +def call_with_retry(function, func_args, max_tries=5): + for attempt in range(max_tries): + try: + result = function(*func_args) + + return result + except Exception: + logger.exception("Error occurred while calling LLM function") + if attempt == max_tries - 1: + raise + + # TODO: is this okay + delay = 10 + delay *= random.uniform(0.5, 1.5) # jitter + time.sleep(delay) + + + +def get_api_key(provider): + key_map = { + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "groq": "GROQ_API_KEY" + } + + key = os.environ.get(key_map[provider]) + if key is None: + raise ValueError(f"Could not find API key for {provider}") + return key + + + + + + + diff --git a/app/db_utils/data_profile.py b/app/db_utils/data_profile.py index 68a981b..4a8c190 100644 --- a/app/db_utils/data_profile.py +++ b/app/db_utils/data_profile.py @@ -3,6 +3,7 @@ import json from pandas.core.arrays import categorical +from app import db_operations, logger from app.db_utils.execute_sql import fetch_sql from app.db_utils.column_types import ColumnTypes @@ -83,11 +84,8 @@ def get_col_names(self): if row[0] not in ['index', 'level_0', ]: col_names.append(row[0]) - print("COL NAMES FROM QUERY: ", col_names) - - except Exception as e: - print(f"AHHHHHHHHHH Querying for col names unsuccessful because of error: {e}") + logger.exception("Error when querying for col names") return col_names @@ -105,7 +103,7 @@ def look_up_stat_from_profile(self, attribute_name, column_name): return stat except Exception as e: - print(f"Error querying attribute from data profile table: {e}") + logger.exception(f"Error querying attribute from data profile table") return None @@ -137,7 +135,7 @@ def calculate_column_attribute(self, attribute_name, column_name, look_up_stat=T :param look_up_stat: If True, first attempt to look up the statistic from the data profile table. If False, calculate it directly. :return: The value of the statistic if found or calculated, otherwise None. """ - if look_up_stat: + if look_up_stat and db_operations.table_exists(self.data_profile_table_name): look_up_value = self.look_up_stat_from_profile(attribute_name, column_name) if look_up_value is not None: @@ -160,7 +158,7 @@ def _calculate_mean(self, column_name): try: avg = self.calculate_summary_stat_using_sql('AVG', column_name) except Exception as e: - print(f"Error fetching the mean for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the mean for table {self.table_name} at column {column_name}") avg = None @@ -183,7 +181,7 @@ def _calculate_median(self, column_name): query += f' WHERE pg_input_is_valid("{column_name}", \'numeric\')' median = fetch_sql(query, True, self.engine) except Exception as e: - print(f"Error fetching the median for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the median for table {self.table_name} at column {column_name}") median = float('nan') return median @@ -199,7 +197,7 @@ def _calculate_max(self, column_name): try: maximum = self.calculate_summary_stat_using_sql('MAX', column_name) except Exception as e: - print(f"Error fetching the maximum for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the maximum for table {self.table_name} at column {column_name}") maximum = None @@ -215,7 +213,7 @@ def _calculate_min(self, column_name): try: minimum = self.calculate_summary_stat_using_sql('MIN', column_name) except Exception as e: - print(f"Error fetching the minimum for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the minimum for table {self.table_name} at column {column_name}") minimum = None @@ -233,8 +231,8 @@ def _calculate_num_categories(self, column_name): n_categories = fetch_sql(query, True, self.engine) except Exception as e: - print("AHHH SQL QUERY DIDN'T WORK") - print(f"Error fetching the n_categories for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the n_categories for table {self.table_name} at column {column_name}") + n_categories = None @@ -259,8 +257,7 @@ def _calculate_mode(self, column_name): print("MODE FROM SQL QUERY: ", mode) except Exception as e: - print("AHHH SQL QUERY DIDN'T WORK") - print(f"Error fetching the mode for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the mode for table {self.table_name} at column {column_name}") mode = None @@ -288,7 +285,7 @@ def _calculate_error_count_dict(self, column_name): error_counts = json.dumps({}) except Exception as e: - print(f"Error fetching the error counts for table {self.table_name} at column {column_name}: {e}") + logger.exception(f"Error fetching the error counts for table {self.table_name} at column {column_name}") error_counts = None @@ -338,23 +335,34 @@ def _calculate_category_count_dict(self, column_name): """ try: - query = f""" - SELECT "{column_name}", COUNT(*) - FROM "{self.table_name}" - GROUP BY "{column_name}" - """ - rows = fetch_sql(query, False ,self.engine) - category_counts = {} - # Put the results into a dict - for (category, count) in rows: - category_counts[category] = count - print("CATEGORY COUNTS DICT", category_counts) + n_categories = self.calculate_column_attribute("n_categories", column_name, True) + n_rows_query = f""" + SELECT "{column_name}", COUNT(*) FROM "{self.table_name}" + """ + n_rows = fetch_sql(n_rows_query, True, self.engine) - except Exception as e: + # Don't wanna calculate category counts if each "category" is unique + if not n_categories == n_rows: + + query = f""" + SELECT "{column_name}", COUNT(*) + FROM "{self.table_name}" + GROUP BY "{column_name}" + """ - print(f"Error fetching the category counts for table {self.table_name} at column {column_name}: {e}") + rows = fetch_sql(query, False ,self.engine) + category_counts = {} + # Put the results into a dict + for (category, count) in rows: + category_counts[category] = count + else: + category_counts = None + + + except Exception as e: + logger.exception(f"Error fetching the category counts for table {self.table_name} at column {column_name}") category_counts = None if category_counts is not None: @@ -373,19 +381,20 @@ def _calculate_class_error_count_dict(self, column_name): # TODO: Implement SQL query version print("Calculating class error counts manually using data...") - self.load_error_df() - - counts_by_column = {} - if not self._error_df.empty: # If error_df is empty (no errors in data selection) - counts_by_column = ( - self._error_df.groupby(['column_id', 'error_type']) - .size() - .unstack(fill_value=0) - .to_dict(orient='index') - ) + try: + counts_by_column = {} + if not self._error_df.empty: # If error_df is empty (no errors in data selection) + counts_by_column = ( + self._error_df.groupby(['column_id', 'error_type']) + .size() + .unstack(fill_value=0) + .to_dict(orient='index') + ) - if counts_by_column is not None: - counts_by_column = json.dumps(counts_by_column) + if counts_by_column is not None: + counts_by_column = json.dumps(counts_by_column) - return counts_by_column + return counts_by_column + except Exception as e: + logger.exception(f"Error fetching the class error counts for table {self.table_name}") diff --git a/app/db_utils/db_functions_sql.py b/app/db_utils/db_functions_sql.py index 9c07537..3309aab 100644 --- a/app/db_utils/db_functions_sql.py +++ b/app/db_utils/db_functions_sql.py @@ -30,6 +30,7 @@ def __init__(self, engine): :param engine: SQLAlchemy engine """ self.engine = engine + self.base_table_name = None self.main_table_name = None self.error_table_name = None self.dp_table_name = None @@ -42,6 +43,7 @@ def reset(self): Resets the DBOperations state, clearing all loaded table references. Called when the user navigates back to the home page. """ + self.base_table_name = None self.main_table_name = None self.error_table_name = None self.dp_table_name = None @@ -49,19 +51,27 @@ def reset(self): self.filtering_table = None self.active_hists = {} - def load_table(self, main_table_name: str, error_table_name: str = None, dp_table_name: str = None): + def load_table(self, main_table_name: str, error_table_name: str = None, dp_table_name: str = None, base_table_name: str = None): """ Loads in the main and error tables, inits the ColumnTypes and FilteringSQL objects with the new table :param main_table_name: the name of the table in the database without errors detected (raw data) :param error_table_name: explicit errors table name; defaults to "errors_" + main_table_name """ + self.main_table_name = main_table_name self.error_table_name = error_table_name if error_table_name is not None else "errors_" + main_table_name self.dp_table_name = dp_table_name if dp_table_name is not None else "dp_" + main_table_name self.col_types = ColumnTypes(main_table_name, self.engine) self.filtering_table = FilteringSQL(main_table_name, self.engine) + assert self.filtering_table is not None self.active_hists = {} + print("LOADED TABLE!!!") + + if base_table_name is not None: + self.base_table_name = base_table_name + + print("FINISHED LOADING TABLE") def get_row_count(self, table_name: str) -> int: """ @@ -152,6 +162,10 @@ def update_active_hists(self, binned_data: list, one_dim: bool, col_key): rows_to_bins = {} bins_to_rows = defaultdict(list) + if binned_data is None: + # Will happen if all of the data happens to be deleted (all values are Null in a column, etc.) + # Setting to empty list so it won't show error when it tries to loop through a None + binned_data = [] if one_dim: for row_id, row_bin in binned_data: diff --git a/app/db_utils/execute_sql.py b/app/db_utils/execute_sql.py index f80cf96..b980727 100644 --- a/app/db_utils/execute_sql.py +++ b/app/db_utils/execute_sql.py @@ -4,13 +4,13 @@ Thin wrapper around SQLAlchemy for executing and fetching results from a PostgreSQL database. """ -def execute_sql(query: str, engine): +def execute_sql(query: str, engine, params=None): """ Executes given SQL query to the postgres database. """ with engine.begin() as conn: - conn.execute(text(query)) + conn.execute(text(query), params) def fetch_sql(query: str, scalar: bool, engine, params=None): """ @@ -33,3 +33,17 @@ def fetch_sql(query: str, scalar: bool, engine, params=None): return None +def copy_table_to_csv(table_name: str, csv_file_path: str, engine): + """ + Copies the contents of a PostgreSQL table to a CSV file. + :arg: table_name: name of the table to copy. + :arg: csv_file_path: path to the CSV file to write to. + """ + print("table_name", table_name) + print("csv_file_path", csv_file_path) + query = f'COPY "{table_name}" TO STDOUT WITH CSV HEADER' + with engine.raw_connection() as conn: + with open(csv_file_path, 'w') as f: + cursor = conn.cursor() + cursor.copy_expert(query, f) + cursor.close() \ No newline at end of file diff --git a/app/routes/ai_routes.py b/app/routes/ai_routes.py new file mode 100644 index 0000000..9542cab --- /dev/null +++ b/app/routes/ai_routes.py @@ -0,0 +1,306 @@ + +from litellm import completion, get_max_tokens, token_counter +from app import app, engine, logger +from app.db_utils.ai_utils import update_csvs_for_llm, get_api_key, parse_json_response +from app.db_utils.execute_sql import fetch_sql, execute_sql + +from app.db_utils.ai_utils import call_with_retry + +AI_SETTINGS_TABLE_NAME = "ai_settings" +from flask import request + +ablations = ["include_data_profile", "include_dataset_context", "include_action_plan", "include_action_plan_translation"] +valid_actions = ["delete_wrangle", "impute_wrangle", "delete_column", "plan_end"] + +''' + LLM Query plans: + - just the action plan in json + - action plan in text -> translate to json + - +''' + + + +# TODO: either put this back to action_details or apply rows and columns to the rest of the logs + +def query_llm_for_text_action_plan(model, provider, api_key, error_log_csv_path, action_log_csv_path, + data_profile_csv_path, full_dataset_csv_path, action_limit=5): + system_prompt = f"You are data scientist. Create an action plan of the top {action_limit} steps the user could do that efficiently cleans this dataset." + + csv_text = {} + + with open(action_log_csv_path, "r") as f: + csv_text[action_log_csv_path] = f.read() + + with open(data_profile_csv_path, "r") as f: + csv_text[data_profile_csv_path] = f.read() + + with open(error_log_csv_path, "r") as f: + csv_text[error_log_csv_path] = f.read() + + with open(full_dataset_csv_path, "r") as f: + csv_text[full_dataset_csv_path] = f.read() + + llm_text_plan_rules = ''' + Each step of the action plan should be numbered and begin with the selection of columns and rows + to perform the wrangles on. You may also select the whole dataset by putting "ALL" for "rows" and "columns". At each step, determine a SINGULAR wrangling + action to perform on the selected data. Here are the wrangles that you can perform on the data: + - Delete: deletes the selection of data + - Impute: impute the selection of data with either the mean if numeric or the mode if categorical + - Delete column: deletes a column of the dataset + - Stop: stop wrangling the data as the data is at a satisfactory state. This does not have to be the last action in the plan. + The stop action should only be used if there are no more actions needed to be done. + + OTHER RULES THAT YOU MUST FOLLOW + - DO NOT CREATE NEW ACTIONS + - ONLY OUTPUT THE ACTION PLAN, NOTHING ELSE + ''' + + additional_details = f''' Here are the contents of each of the CSVs that you can use to inform the actions in your action plan: + action_log: {csv_text[action_log_csv_path]}\n + data_profile: {csv_text[data_profile_csv_path]}\n + error_log: {csv_text[error_log_csv_path]}\n + full_dataset: {csv_text[full_dataset_csv_path]}\n + ''' + + full_message = llm_text_plan_rules + additional_details + print("full_message", full_message) + print("len(full_message)", len(full_message)) + print("estimated num tokens", len(full_message) / 4) + full_model_name = provider + '/' + model + print("MAX TOKENS", get_max_tokens(full_model_name)) + print("TOKEN COUNT", token_counter(model=full_model_name, text=full_message)) + + response = completion( + model=full_model_name, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": full_message}, + ], + api_key=api_key + ) + + llm_text_response = response.choices[0].message.content + print("LLM TEXT RESPONSE", llm_text_response) + + return llm_text_response + + +def query_llm_for_action_plan_translation(model, provider, api_key, text_action_plan): + system_prompt = "You are a translator that translates actions from natural language text to JSON where each row is an action. Translate this action plan from text to JSON." + + llm_translated_plan_rules = ''' + Each step of the action plan should be a dict with the following keys: "action_name", "rows_to_wrangle", "column". + Here are the valid actions and their names: + - Delete -> "delete_wrangle" + - Impute -> "impute_wrangle" + - Delete column -> "delete_column" + - Stop -> "plan_end" + + The value of "rows_to_wrangle" is list of rows to wrangle. + The value of "column" is the name of a SINGLE column to wrangle. + + OTHER RULES THAT YOU MUST FOLLOW + - DO NOT CREATE NEW ACTIONS + - THE RESULT MUST BE IN JSON FORMAT + ''' + + text_action_plan_prompt = f''' + Here is the text action plan you must translate to JSON: + {text_action_plan} + ''' + + response = completion( + model=provider + '/' + model, + response_format={"type": "json_object"}, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": llm_translated_plan_rules + text_action_plan_prompt}, + ], + api_key=api_key + ) + + llm_text_response = response.choices[0].message.content + print("LLM TEXT TRANSLATION RESPONSE", llm_text_response) + + json_action_plan = parse_json_response(llm_text_response) + + return json_action_plan + + +def query_llm_for_dataset_context(model, api_key, column_names, user_provided_dataset_context, dataset_name): + system_prompt = ("You are a data scientist that is given a dataset. " + "You are to provide context about the dataset. You are to give a description of the dataset as well" + "as a brief description of each column. If you do not know what a column is, you can have the" + "description be 'N/A'") + + llm_dataset_context_rules = f''' + The dataset is named: {dataset_name} + The columns in the dataset are: {column_names} + The user has provided the following context about the dataset: {user_provided_dataset_context} + + Using this information, create a JSON object with the following keys: 'dataset_description', 'column_descriptions'. + The value of 'dataset_description' is the description of the dataset. + The value of 'column_descriptions' is a dictionary where the keys are the column names and the values are the descriptions of each column. + If you do not know what a column is, you can have the description be None. + ''' + + response = completion( + model=model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": llm_dataset_context_rules}, + ], + api_key=api_key + ) + + llm_text_response = response.choices[0].message.content + return llm_text_response + +@app.post('/api/ai_helper/perform_llm_action') +def perform_llm_action(): + from app import db_operations + from app.db_utils.query import remove_rows_by_ids, impute_by_ids + action_dict = request.get_json() + + try: + main_table_name = db_operations.main_table_name + action_name = action_dict["action_name"] + rows_to_wrangle = action_dict["rows_to_wrangle"] + column = action_dict["column"] + + if action_name == "stop": + return { + "success": True + } + + # Apply the changes directly to the table (no preview) + if action_name == "delete": + remove_rows_by_ids(table=main_table_name, ids=rows_to_wrangle) + elif action_name == "impute": + impute_by_ids(table=main_table_name, col=column,ids=rows_to_wrangle) + + return { + "success": True + } + + + except Exception as e: + logger.exception("Error performing LLM action") + return { + "success": False + } + +@app.post('/api/ai_helper/get_action_plan') +def get_llm_json_action_plan(): + from app import db_operations, engine + from app.server_utils.logger_utils import ACTION_LOG_TABLE_NAME + try: + + error_table_name = db_operations.error_table_name + data_profile_name = db_operations.dp_table_name + full_dataset_name = db_operations.main_table_name + action_log_name = ACTION_LOG_TABLE_NAME + settings_dict = get_settings_dict(engine) + model_name = settings_dict.get("model_name") + provider = settings_dict.get("provider") + api_key = get_api_key(provider) + + assert model_name is not None + + (error_log_csv_path, data_profile_csv_path, action_log_csv_path, full_dataset_csv_path) = update_csvs_for_llm(error_table_name, + data_profile_name, + action_log_name, full_dataset_name) + + text_plan_func_args = (model_name, provider, api_key, action_log_csv_path, + error_log_csv_path, data_profile_csv_path, full_dataset_csv_path) + text_action_plan = call_with_retry(query_llm_for_text_action_plan, text_plan_func_args, max_tries=5) + + translation_func_args = (model_name, provider, api_key, text_action_plan) + + json_action_plan = call_with_retry(query_llm_for_action_plan_translation, translation_func_args, + max_tries=5) + + return {"success": True, "json_action_plan": json_action_plan} + except Exception as e: + json_action_plan = None + logger.exception("Error translating llm action plan to json") + return {"success": False, "json_action_plan": json_action_plan} + + +@app.post('/api/ai_helper/update_settings_table') +def update_settings_table(): + print("UPDATING SETTINGS TABLE") + data = request.get_json() + model_name = data.get("model_name") + provider = data.get("provider") + print("MODEL NAME", model_name, "PROVIDER", provider) + + try: + execute_sql(f""" + CREATE TABLE IF NOT EXISTS {AI_SETTINGS_TABLE_NAME} + ( + id + INTEGER + PRIMARY + KEY, + model_name + TEXT + NOT + NULL, + provider + TEXT + NOT + NULL + ) + """, engine) + + result = fetch_sql(f"SELECT id FROM {AI_SETTINGS_TABLE_NAME} WHERE id = :id", True, + engine, {"id": 1} + ) + + if result is None: + print("VALUE DOESN'T EXIST. INSERTING ONE") + execute_sql( + f"INSERT INTO {AI_SETTINGS_TABLE_NAME} (id, model_name, provider) VALUES (:id, :model_name, :provider)", + engine, {"id": 1, "model_name": model_name, "provider": provider} + ) + else: + print("VALUE EXISTS, REPLACING") + execute_sql( + f"UPDATE {AI_SETTINGS_TABLE_NAME} SET model_name = :model_name, provider = :provider WHERE id = :id", + engine, {"id": 1, "model_name": model_name, "provider": provider} + ) + + print("MODEL NAME", model_name, "PROVIDER", provider) + return {"success": True} + except Exception as e: + logger.exception("Error updating settings table.") + + return {"success": False} + +# TODO: what do i do if get_settings_dict is None? +def get_settings_dict(engine): + try: + result = fetch_sql(f"SELECT model_name, provider FROM {AI_SETTINGS_TABLE_NAME} WHERE id = :id", False, engine, {"id": 1}) + row = result[0] + + if result is not None: + model_name = row.model_name # or row[0] + provider = row.provider + print("MODEL NAME", model_name, "PROVIDER", provider) + # TODO: maybe don't return a dict + settings_dict = {"model_name": model_name, "provider": provider} + else: + settings_dict = None + + return settings_dict + except Exception as e: + logger.exception("Error retrieving settings from table.") + raise + + + + + + diff --git a/app/routes/plot_routes.py b/app/routes/plot_routes.py index c8c97cc..10a565f 100644 --- a/app/routes/plot_routes.py +++ b/app/routes/plot_routes.py @@ -255,6 +255,7 @@ def get_preview_histogram(): x_bins = int(request.args.get("x_bins", 10)) y_bins = int(request.args.get("y_bins", 10)) histogram = preview_ops.generate_two_d_histogram_with_errors(column_x, column_y, x_bins, y_bins) + print("PREVIEW OPS FILTERING TABLE!!!!", preview_ops.filtering_table) return {"success": True, "histogram": histogram} except Exception as e: diff --git a/app/routes/routes.py b/app/routes/routes.py index d65ad9c..b96d563 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -9,12 +9,14 @@ from app import db_operations, engine from app.db_utils.data_profile import DataProfile from app.server_utils.service_helpers import ( - generate_table_name, + generate_base_table_name, create_error_df, get_sqlalchemy_dtype_map, calculate_attribute_rankings, get_pgraph_redo, get_pgraph_undo, init_pgraph_for_session, create_data_profile_df, ) +from datetime import datetime, timezone from app.server_utils.set_id_column import set_id_column +from app.server_utils.logger_utils import update_action_log, initialize_action_log, initialize_preview_log_table def load_file(csv_file, filename): @@ -30,6 +32,7 @@ def load_file(csv_file, filename): :param filename: the name of the csv_file :return: json object """ + timestamp = datetime.now(timezone.utc) dataframe = pd.read_csv(csv_file) # run the detectors on the uploaded file for the starting data state @@ -38,8 +41,13 @@ def load_file(csv_file, filename): detected_data = create_error_df(dataframe) time_to_detect = time.time() - start_time app.original_table_name = filename - table_name = generate_table_name(filename) - table_name_with_node_id = f"n0_{table_name}" + base_table_name = generate_base_table_name(filename) + + initialize_action_log(engine) + initialize_preview_log_table(engine) + + + table_name_with_node_id = f"n0_{base_table_name}" # Build dtype map from actual column values before pushing to DB dtype_map = get_sqlalchemy_dtype_map(table_with_id_added) error_table_name = f"errors_{table_name_with_node_id}" @@ -67,7 +75,8 @@ def load_file(csv_file, filename): now we fully init the DBOperations object that was first initialized in init.py, get the actual row counts since .to_sql is buggy and not right """ - db_operations.load_table(table_name_with_node_id, error_table_name, dp_table_name) + db_operations.load_table(table_name_with_node_id, error_table_name, dp_table_name, base_table_name=base_table_name) + print("DB OPERATIONS LOAD_TABLE DONE") rows_affected = db_operations.get_row_count(table_name_with_node_id) detected_rows_affected = db_operations.get_row_count(error_table_name) @@ -77,13 +86,20 @@ def load_file(csv_file, filename): #init the pgraph init_pgraph_for_session(table_name_with_node_id) + action_duration = (datetime.now(timezone.utc) - timestamp).total_seconds() + update_action_log(dataset_id=base_table_name, action_name="load_dataset", action_details=None, engine=engine, + timestamp=timestamp, action_duration=action_duration, action_successful=True) return {"success": True, "rows for undetected data": rows_affected, "rows_for_detected": detected_rows_affected, "table_name": table_name_with_node_id} except Exception as e: print(f"Error in upload: {e}") import traceback traceback.print_exc() + + update_action_log(dataset_id=base_table_name, action_name="load_dataset", action_details=None, engine=engine, + timestamp=timestamp, action_successful=False, + action_error_message=e) return {"success": False, "error": str(e)} diff --git a/app/routes/wrangler_routes_sql.py b/app/routes/wrangler_routes_sql.py index 5e5fe8b..ca9f6df 100644 --- a/app/routes/wrangler_routes_sql.py +++ b/app/routes/wrangler_routes_sql.py @@ -2,27 +2,23 @@ # This file handles all endpoints surrounding wranglers from flask import request -from app import app, db_operations from app.db_utils import query -from app import engine +from app import app, engine, db_operations import traceback import pandas as pd from app.server_utils.service_helpers import create_error_df, create_previews_1d, create_previews_2d, \ - execute_wrangle_preview, _safe_pg_name, create_data_profile_df, get_sqlalchemy_dtype_map + execute_wrangle_preview, _safe_pg_name, create_data_profile_df, get_sqlalchemy_dtype_map, extract_preview_action from sqlalchemy import inspect, text from app.db_utils.data_profile import DataProfile +from datetime import datetime, timezone +from app.server_utils.logger_utils import update_action_log, update_preview_log, get_action_details_from_preview_log +import json """ Wrangling Endpoints - In-place modification of tables """ -def get_table_dtypes(target_table_name, engine): - """Build a dtype dict for to_sql() by reflecting the target table's real column types.""" - inspector = inspect(engine) - columns = inspector.get_columns(target_table_name) - # col["type"] is already a SQLAlchemy type instance we can hand straight to to_sql - return {col["name"]: col["type"] for col in columns} # Where updated_df is just the data that needed to actually be updated # Assumes that updated_df has the same columns as the target table @@ -59,6 +55,17 @@ def update_table(updated_df, target_table_name, key_col, cols_to_remove): # Helper: Re-run error detection after modification # ───────────────────────────────────────────────────────────────────────────── +#def mark_dirty_rows_data_profile(table_name, col_names): + + +#def mark_dirty_rows_errors_table(table_name, col_names): + + +# TODO: Finish this later +#def update_table_rows(table_name, col_names: list) -> None: + + + # Returns error_df for update_data_profile_table to use (so it doesn't have to get it from the database) def update_errors_table(table_name: str, columns_selected_for_wrangling: list) -> pd.DataFrame: # TODO: fix this so it doesn't update the whole table after small changes to the table @@ -87,7 +94,6 @@ def update_errors_table(table_name: str, columns_selected_for_wrangling: list) - # detected_errors_df.to_sql(errors_table_name, engine, if_exists='fail', index=False) print(f"✓ Updated errors table: {errors_table_name}") - return detected_errors_df except Exception as e: print(f"ERROR: Could not update errors table for {table_name}: {e}") traceback.print_exc() @@ -120,15 +126,6 @@ def update_data_profile_table(table_name: str, columns_selected_for_wrangling: l raise -# TODO: Finish this later -#def update_stat_to_data_profile_table(table_name: str, error_df: pd.DataFrame) -> None: - - - - - - -# TODO: does this even do anything? Can I remove it? def update_preview_error_table(table_name: str, err_table_name: str) -> None: """ After modifying a table in-place, re-run error detection @@ -151,6 +148,17 @@ def update_preview_error_table(table_name: str, err_table_name: str) -> None: traceback.print_exc() raise +def execute_wrangle_logic(preview_table, table): + action_details_dict = get_action_details_from_preview_log(preview_table, engine) + + wrangle_executed = extract_preview_action(preview_table) + + new_table_name = execute_wrangle_preview(table, preview_table, _safe_pg_name, db_operations) + + return (new_table_name, action_details_dict, wrangle_executed) + + + # ───────────────────────────────────────────────────────────────────────────── # Wrangling Endpoints (Supports both bin-based and ID-based selections) # the way it works is, create-previews does all wrangles (delete, impute x/y), @@ -178,24 +186,73 @@ def create_previews(): row_ids – list of integer row IDs to operate on cols – list of column names involved in the selection (for imputation) """ + + timestamp = datetime.now(timezone.utc) try: body = request.get_json(force=True) table = db_operations.main_table_name row_ids = body.get("row_ids", []) cols = body.get("cols", []) + # extra case protection. #cols = [f'{col}' for col in cols] if not row_ids: + update_action_log(dataset_id=table, action_name="create_previews", + action_details=json.dumps({"row_ids": row_ids, "cols": cols}), engine=engine, + timestamp=timestamp, action_successful=False, action_error_message="Row IDs list is empty") + return {"success": False, "error": "No rows selected"}, 400 + + action_details_dict = {"row_ids": row_ids, "cols": cols} + if len(cols) == 1: - return create_previews_1d(table, row_ids, cols, _safe_pg_name, update_errors_table, update_data_profile_table) + + (preview_delete_table_name, preview_impute_table_name) = create_previews_1d(table, row_ids, cols, _safe_pg_name, update_errors_table, update_data_profile_table) + + update_preview_log(preview_delete_table_name, "delete_wrangle", action_details_dict, engine) + update_preview_log(preview_impute_table_name, "impute_wrangle", action_details_dict, engine) + + result_dict = { + "success": True, + "preview_delete": preview_delete_table_name, + "preview_impute": preview_impute_table_name, + "dims": 1, + } else: - return create_previews_2d(table, row_ids, cols, _safe_pg_name, update_errors_table, update_data_profile_table) + (preview_delete_table_name, preview_impute_x_table_name, preview_impute_y_table_name) = create_previews_2d(table, row_ids, cols, _safe_pg_name, update_errors_table, update_data_profile_table) + + update_preview_log(preview_delete_table_name, "delete_wrangle", action_details_dict, engine) + update_preview_log(preview_impute_x_table_name, "impute_x_wrangle", action_details_dict, engine) + update_preview_log(preview_impute_y_table_name, "impute_y_wrangle", action_details_dict, engine) + + result_dict = { + "success": True, + "preview_delete": preview_delete_table_name, + "preview_impute_x": preview_impute_x_table_name, + "preview_impute_y": preview_impute_y_table_name, + "dims": 2, + } + + action_duration = (datetime.now(timezone.utc) - timestamp).total_seconds() + + + update_action_log(dataset_id=table, action_name="create_previews", + action_details=json.dumps(action_details_dict), engine=engine, + timestamp=timestamp, action_duration= action_duration,action_successful=True) + + assert result_dict is not None + + return result_dict except Exception as e: + + update_action_log(dataset_id=table, action_name="create_previews", + action_details=json.dumps({"row_ids": row_ids, "cols": cols}), engine=engine, + timestamp=timestamp, action_successful=False, action_error_message=e) + print("ERROR in create_previews") print(traceback.format_exc()) return {"success": False, "error": str(e)}, 400 @@ -210,25 +267,39 @@ def execute_wrangle(): 3. Rename the selected preview table to