From 19ba6d6b0fb4bf5d4f50675e71c991be34faef12 Mon Sep 17 00:00:00 2001 From: Alex Happy <1223408988@qq.com> Date: Fri, 11 Sep 2026 15:10:54 +0800 Subject: [PATCH] optimize excel update import: query only needed columns instead of select * --- dtable_events/__init__.py | 2 +- dtable_events/dtable_io/excel.py | 30 ++++++++++++++++++++++++------ dtable_events/dtable_io/utils.py | 8 ++++++-- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/dtable_events/__init__.py b/dtable_events/__init__.py index ffcfdc3e..77a3193f 100644 --- a/dtable_events/__init__.py +++ b/dtable_events/__init__.py @@ -2,7 +2,7 @@ from dtable_events.activities.db import get_table_activities, get_activities_detail from dtable_events.app.event_redis import RedisClient from dtable_events.statistics.db import get_user_activity_stats_by_day, get_daily_active_users, get_email_sending_logs -from dtable_events.dtable_io.excel import get_insert_update_rows +from dtable_events.dtable_io.excel import get_insert_update_rows, get_import_update_query_columns from dtable_events.dtable_io.utils import update_page_design_static_image, rename_universal_app_static_assets_dir, \ update_universal_app_custom_page_static_image, update_universal_app_single_record_page_static_assets from dtable_events.utils.sql_generator import filter2sql, statistic2sql, linkRecords2sql, SQLGeneratorOptionInvalidError, \ diff --git a/dtable_events/dtable_io/excel.py b/dtable_events/dtable_io/excel.py index 5ff18bbe..ee81b2a9 100644 --- a/dtable_events/dtable_io/excel.py +++ b/dtable_events/dtable_io/excel.py @@ -531,6 +531,7 @@ def parse_and_update_file_to_table(file_name, username, dtable_uuid, table_name, dtable_server_api = DTableServerAPI(username, dtable_uuid, INNER_DTABLE_SERVER_URL) columns = dtable_server_api.list_columns(table_name) + dtable_col_name_to_column = {col['name']: col for col in columns} try: # file_type is xlsx or csv @@ -547,9 +548,8 @@ def parse_and_update_file_to_table(file_name, username, dtable_uuid, table_name, key_columns = selected_columns.split(',') dtable_db_api = DTableDBAPI(username, dtable_uuid, INNER_DTABLE_DB_URL) - dtable_rows = get_rows_from_dtable_db(dtable_db_api, table_name) - - dtable_col_name_to_column = {col['name']: col for col in columns} + query_columns = get_import_update_query_columns(dtable_col_name_to_column, file_rows, key_columns) + dtable_rows = get_rows_from_dtable_db(dtable_db_api, table_name, columns=query_columns) insert_rows, update_rows, excel_select_column_options = \ get_insert_update_rows(dtable_col_name_to_column, file_rows, dtable_rows, key_columns, need_select_option=True) @@ -778,15 +778,16 @@ def update_parsed_file_by_dtable_server(username, dtable_uuid, file_name, table_ excel_rows = excel_rows[0].get('rows', []) key_columns = selected_columns.split(',') - dtable_db_api = DTableDBAPI(username, dtable_uuid, INNER_DTABLE_DB_URL) - dtable_rows = get_rows_from_dtable_db(dtable_db_api, table_name) - dtable_server_api = DTableServerAPI(username, dtable_uuid, INNER_DTABLE_SERVER_URL) columns = dtable_server_api.list_columns(table_name) dtable_col_name_to_column = {col['name']: col for col in columns} + dtable_db_api = DTableDBAPI(username, dtable_uuid, INNER_DTABLE_DB_URL) + query_columns = get_import_update_query_columns(dtable_col_name_to_column, excel_rows, key_columns) + dtable_rows = get_rows_from_dtable_db(dtable_db_api, table_name, columns=query_columns) + insert_rows, update_rows, excel_select_column_options = \ get_insert_update_rows(dtable_col_name_to_column, excel_rows, dtable_rows, key_columns, need_select_option=True) @@ -817,6 +818,23 @@ def get_cell_value(row, col, excel_col_name_to_type): return cell_value +def get_import_update_query_columns(dtable_col_name_to_column, excel_rows, key_columns): + """Return the minimal column names needed from dtable-db to decide update vs append. + + The update/append decision only needs the key columns (used to match rows) and the + matched columns whose type is in UPDATE_TYPE_LIST (used to diff cells). The row id is + also required to update existing rows. + """ + columns = set(key_columns) + if excel_rows: + for col_name in excel_rows[0].keys(): + col = dtable_col_name_to_column.get(col_name) + if col and col.get('type') in UPDATE_TYPE_LIST: + columns.add(col_name) + columns.add('_id') + return [col for col in columns if col == '_id' or col in dtable_col_name_to_column] + + def get_insert_update_rows(dtable_col_name_to_column, excel_rows, dtable_rows, key_columns, need_select_option=False): if not excel_rows: return [], [], {} diff --git a/dtable_events/dtable_io/utils.py b/dtable_events/dtable_io/utils.py index af9482fd..8d9eabae 100644 --- a/dtable_events/dtable_io/utils.py +++ b/dtable_events/dtable_io/utils.py @@ -1512,7 +1512,7 @@ def to_python_boolean(string): raise ValueError("Invalid boolean value: '%s'" % string) -def get_rows_from_dtable_db(dtable_db_api, table_name, limit=50000): +def get_rows_from_dtable_db(dtable_db_api, table_name, limit=50000, columns=None): from dtable_events.utils.dtable_db_api import convert_db_rows offset = 10000 start = 0 @@ -1522,7 +1522,11 @@ def get_rows_from_dtable_db(dtable_db_api, table_name, limit=50000): if (start + offset) > limit: offset = limit - start - sql = f"SELECT * FROM `{table_name}` LIMIT {start}, {offset}" + if columns: + column_names_str = ', '.join('`%s`' % name for name in columns) + sql = f"SELECT {column_names_str} FROM `{table_name}` LIMIT {start}, {offset}" + else: + sql = f"SELECT * FROM `{table_name}` LIMIT {start}, {offset}" response_rows, metadata = dtable_db_api.query(sql, convert=False, server_only=True) response_rows = convert_db_rows(metadata, response_rows)