Skip to content

optimize excel update import: query only needed columns instead of select * #910

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AlexCXC wants to merge 1 commit into
base: master
Choose a base branch
from
Open

optimize excel update import: query only needed columns instead of select * #910

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dtable_events/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from dtable_events.activities.db import get_table_activities, get_activities_detail
from dtable_events.app.event_redis import RedisClient
from dtable_events.statistics.db import get_user_activity_stats_by_day, get_daily_active_users, get_email_sending_logs
from dtable_events.dtable_io.excel import get_insert_update_rows
from dtable_events.dtable_io.excel import get_insert_update_rows, get_import_update_query_columns
from dtable_events.dtable_io.utils import update_page_design_static_image, rename_universal_app_static_assets_dir, \
update_universal_app_custom_page_static_image, update_universal_app_single_record_page_static_assets
from dtable_events.utils.sql_generator import filter2sql, statistic2sql, linkRecords2sql, SQLGeneratorOptionInvalidError, \
Expand Down
30 changes: 24 additions & 6 deletions dtable_events/dtable_io/excel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -531,6 +531,7 @@ def parse_and_update_file_to_table(file_name, username, dtable_uuid, table_name,

dtable_server_api = DTableServerAPI(username, dtable_uuid, INNER_DTABLE_SERVER_URL)
columns = dtable_server_api.list_columns(table_name)
dtable_col_name_to_column = {col['name']: col for col in columns}

try:
# file_type is xlsx or csv
Expand All @@ -547,9 +548,8 @@ def parse_and_update_file_to_table(file_name, username, dtable_uuid, table_name,
key_columns = selected_columns.split(',')

dtable_db_api = DTableDBAPI(username, dtable_uuid, INNER_DTABLE_DB_URL)
dtable_rows = get_rows_from_dtable_db(dtable_db_api, table_name)

dtable_col_name_to_column = {col['name']: col for col in columns}
query_columns = get_import_update_query_columns(dtable_col_name_to_column, file_rows, key_columns)
dtable_rows = get_rows_from_dtable_db(dtable_db_api, table_name, columns=query_columns)

insert_rows, update_rows, excel_select_column_options = \
get_insert_update_rows(dtable_col_name_to_column, file_rows, dtable_rows, key_columns, need_select_option=True)
Expand Down Expand Up @@ -778,15 +778,16 @@ def update_parsed_file_by_dtable_server(username, dtable_uuid, file_name, table_
excel_rows = excel_rows[0].get('rows', [])
key_columns = selected_columns.split(',')

dtable_db_api = DTableDBAPI(username, dtable_uuid, INNER_DTABLE_DB_URL)
dtable_rows = get_rows_from_dtable_db(dtable_db_api, table_name)

dtable_server_api = DTableServerAPI(username, dtable_uuid, INNER_DTABLE_SERVER_URL)

columns = dtable_server_api.list_columns(table_name)

dtable_col_name_to_column = {col['name']: col for col in columns}

dtable_db_api = DTableDBAPI(username, dtable_uuid, INNER_DTABLE_DB_URL)
query_columns = get_import_update_query_columns(dtable_col_name_to_column, excel_rows, key_columns)
dtable_rows = get_rows_from_dtable_db(dtable_db_api, table_name, columns=query_columns)

insert_rows, update_rows, excel_select_column_options = \
get_insert_update_rows(dtable_col_name_to_column, excel_rows, dtable_rows, key_columns, need_select_option=True)

Expand Down Expand Up @@ -817,6 +818,23 @@ def get_cell_value(row, col, excel_col_name_to_type):
return cell_value


def get_import_update_query_columns(dtable_col_name_to_column, excel_rows, key_columns):
"""Return the minimal column names needed from dtable-db to decide update vs append.

The update/append decision only needs the key columns (used to match rows) and the
matched columns whose type is in UPDATE_TYPE_LIST (used to diff cells). The row id is
also required to update existing rows.
"""
columns = set(key_columns)
if excel_rows:
for col_name in excel_rows[0].keys():
col = dtable_col_name_to_column.get(col_name)
if col and col.get('type') in UPDATE_TYPE_LIST:
columns.add(col_name)
columns.add('_id')
return [col for col in columns if col == '_id' or col in dtable_col_name_to_column]


def get_insert_update_rows(dtable_col_name_to_column, excel_rows, dtable_rows, key_columns, need_select_option=False):
if not excel_rows:
return [], [], {}
Expand Down
8 changes: 6 additions & 2 deletions dtable_events/dtable_io/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1512,7 +1512,7 @@ def to_python_boolean(string):
raise ValueError("Invalid boolean value: '%s'" % string)


def get_rows_from_dtable_db(dtable_db_api, table_name, limit=50000):
def get_rows_from_dtable_db(dtable_db_api, table_name, limit=50000, columns=None):

@seafile-dev seafile-dev Sep 11, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Critical] 未将查询总量限制为 1 万

Why this matters:
这里仍保留 limit=50000，循环会按每页 10000 行继续请求至多五页。需求要求正常路径使用 LIMIT 1 万；在大表或复杂公式表中，累计读取 5 万行即使只查询必要列，预检和提交更新仍可能触发 execution cost exceeded，优化目标无法保证达成。

Suggested fix: 将此更新导入路径的总上限改为 10000（或显式传入该值），使正常路径只执行一次 1 万行查询；1 千行降级策略可按任务说明后续实现。

@seafile-dev seafile-dev Sep 11, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

复核更正：任务详情仅要求将查询从 SELECT * 收敛为必要列，并未要求把既有的最多 5 万行扫描上限改为 1 万行。当前实现已将匹配键、可更新导入列和 _id 传入查询；本线程中的“总量限制为 1 万”不应作为本次 PR 的阻塞项。

from dtable_events.utils.dtable_db_api import convert_db_rows
offset = 10000
start = 0
Expand All @@ -1522,7 +1522,11 @@ def get_rows_from_dtable_db(dtable_db_api, table_name, limit=50000):
if (start + offset) > limit:
offset = limit - start

sql = f"SELECT * FROM `{table_name}` LIMIT {start}, {offset}"
if columns:
column_names_str = ', '.join('`%s`' % name for name in columns)
sql = f"SELECT {column_names_str} FROM `{table_name}` LIMIT {start}, {offset}"
else:
sql = f"SELECT * FROM `{table_name}` LIMIT {start}, {offset}"

response_rows, metadata = dtable_db_api.query(sql, convert=False, server_only=True)
response_rows = convert_db_rows(metadata, response_rows)
Expand Down
Loading