Skip to content

export table/view/big-view/app-table to excel limit images total size #907

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
freeplant merged 3 commits into from
Aug 27, 2026
Merged

export table/view/big-view/app-table to excel limit images total size #907

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5623e5f
export table/view/big-view/app-table to excel limit images total size
AlexCXC Aug 21, 2026
5cb8334
update
AlexCXC Aug 22, 2026
9e8b561
update
AlexCXC Aug 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions dtable_events/dtable_io/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -966,10 +966,10 @@ def convert_document_to_pdf(dtable_uuid, doc_uuid, row_id, username):
dtable_io_logger.exception('dtable: %s plugin: document doc_uuid: %s row: %s error: %s', dtable_uuid, doc_uuid, row_id, e)


def convert_view_to_excel(dtable_uuid, table_id, view_id, username, id_in_org, user_department_ids_map, permission, name, repo_id, is_support_image=False):
def convert_view_to_excel(dtable_uuid, table_id, view_id, username, id_in_org, user_department_ids_map, permission, name, repo_id, is_support_image=False, images_max_size=None):
from dtable_events.dtable_io.excel_group import generate_groups, compute_group_summaries
from dtable_events.dtable_io.utils import get_metadata_from_dtable_server, get_export_view_rows_from_dtable_db
from dtable_events.dtable_io.excel import write_xls_with_type, TEMP_EXPORT_VIEW_DIR, IMAGE_TMP_DIR
from dtable_events.dtable_io.excel import write_xls_with_type, TEMP_EXPORT_VIEW_DIR, IMAGE_TMP_DIR, EXPORT_IMAGE_MAX_SIZE
from dtable_events.dtable_io.utils import get_related_nicknames_from_dtable, escape_sheet_name
from dtable_events.utils.dtable_db_api import DTableDBAPI
from dtable_events.utils.sql_generator import pre_filter_to_filter_term
Expand Down Expand Up @@ -1031,7 +1031,9 @@ def convert_view_to_excel(dtable_uuid, table_id, view_id, username, id_in_org, u
summary_col_info.update({col.get('name'): summary_configs.get(col.get('key'))})

images_target_dir = os.path.join(IMAGE_TMP_DIR, dtable_uuid, str(uuid.uuid4()))
image_param = {'num': 0, 'is_support': is_support_image, 'images_target_dir': images_target_dir}
image_param = {'num': 0, 'is_support': is_support_image, 'images_target_dir': images_target_dir,
'total_size': 0, 'max_size': int(images_max_size * 1024 * 1024) if images_max_size else EXPORT_IMAGE_MAX_SIZE,
'stop': False}

sheet_name = table_name + ('_' + view_name if view_name else '')
sheet_name = escape_sheet_name(sheet_name)
Expand Down Expand Up @@ -1096,9 +1098,9 @@ def convert_view_to_excel(dtable_uuid, table_id, view_id, username, id_in_org, u
pass


def convert_table_to_excel(dtable_uuid, table_id, username, name, repo_id, is_support_image=False):
def convert_table_to_excel(dtable_uuid, table_id, username, name, repo_id, is_support_image=False, images_max_size=None):
from dtable_events.dtable_io.utils import get_metadata_from_dtable_server, get_export_table_rows_from_dtable_db
from dtable_events.dtable_io.excel import write_xls_with_type, IMAGE_TMP_DIR
from dtable_events.dtable_io.excel import write_xls_with_type, IMAGE_TMP_DIR, EXPORT_IMAGE_MAX_SIZE
from dtable_events.dtable_io.utils import get_related_nicknames_from_dtable, escape_sheet_name
from dtable_events.utils.dtable_db_api import DTableDBAPI
import openpyxl
Expand Down Expand Up @@ -1146,7 +1148,9 @@ def convert_table_to_excel(dtable_uuid, table_id, username, name, repo_id, is_su
column_name_to_column = {col.get('name'): col for col in cols}

images_target_dir = os.path.join(IMAGE_TMP_DIR, dtable_uuid, str(uuid.uuid4()))
image_param = {'num': 0, 'is_support': is_support_image, 'images_target_dir': images_target_dir}
image_param = {'num': 0, 'is_support': is_support_image, 'images_target_dir': images_target_dir,
'total_size': 0, 'max_size': int(images_max_size * 1024 * 1024) if images_max_size else EXPORT_IMAGE_MAX_SIZE,
'stop': False}

sheet_name = escape_sheet_name(table_name)
excel_name = name + '_' + table_name + '.xlsx'
Expand Down Expand Up @@ -1298,10 +1302,10 @@ def update_big_excel(username, dtable_uuid, table_name, file_path, ref_columns,
dtable_io_logger.info('update big excel %s.xlsx success!' % table_name)


def convert_big_data_view_to_excel(dtable_uuid, table_id, view_id, username, name, task_id, tasks_status_map, repo_id, is_support_image):
def convert_big_data_view_to_excel(dtable_uuid, table_id, view_id, username, name, task_id, tasks_status_map, repo_id, is_support_image, images_max_size=None):
dtable_io_logger.info('Start export big data view to excel: {}.'.format(dtable_uuid))
try:
export_big_data_to_excel(dtable_uuid, table_id, view_id, username, name, task_id, tasks_status_map, repo_id, is_support_image)
export_big_data_to_excel(dtable_uuid, table_id, view_id, username, name, task_id, tasks_status_map, repo_id, is_support_image, images_max_size)
except Exception as e:
dtable_io_logger.error('export big data view failed. ERROR: {}'.format(e))
else:
Expand Down Expand Up @@ -1378,10 +1382,10 @@ def import_page_design(repo_id, workspace_id, dtable_uuid, page_id, is_dir, user
clear_tmp_file(tmp_page_path)


def convert_app_table_page_to_excel(dtable_uuid, repo_id, table_id, username, app_name, page_name, filter_condition_groups, shown_column_keys, task_id, tasks_status_map, is_support_image):
def convert_app_table_page_to_excel(dtable_uuid, repo_id, table_id, username, app_name, page_name, filter_condition_groups, shown_column_keys, task_id, tasks_status_map, is_support_image, images_max_size=None):
dtable_io_logger.info('Start export app table to excel: {}.'.format(dtable_uuid))
try:
export_app_table_page_to_excel(dtable_uuid, repo_id, table_id, username, app_name, page_name, filter_condition_groups, shown_column_keys, task_id, tasks_status_map, is_support_image)
export_app_table_page_to_excel(dtable_uuid, repo_id, table_id, username, app_name, page_name, filter_condition_groups, shown_column_keys, task_id, tasks_status_map, is_support_image, images_max_size)
except Exception as e:
dtable_io_logger.exception('export app table failed. ERROR: {}'.format(e))
else:
Expand Down
14 changes: 9 additions & 5 deletions dtable_events/dtable_io/big_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import uuid
from copy import deepcopy

from dtable_events.dtable_io.excel import parse_row, write_xls_with_type, TEMP_EXPORT_VIEW_DIR, IMAGE_TMP_DIR
from dtable_events.dtable_io.excel import parse_row, write_xls_with_type, TEMP_EXPORT_VIEW_DIR, IMAGE_TMP_DIR, EXPORT_IMAGE_MAX_SIZE
from dtable_events.dtable_io.utils import get_related_nicknames_from_dtable, get_metadata_from_dtable_server, \
escape_sheet_name
from dtable_events.utils import get_location_tree_json, gen_random_option, format_date_in_query
Expand Down Expand Up @@ -439,7 +439,7 @@ def update_excel_to_db(
return


def export_big_data_to_excel(dtable_uuid, table_id, view_id, username, name, task_id, tasks_status_map, repo_id, is_support_image=False):
def export_big_data_to_excel(dtable_uuid, table_id, view_id, username, name, task_id, tasks_status_map, repo_id, is_support_image=False, images_max_size=None):
from dtable_events.dtable_io import dtable_io_logger

# init task_status_map for exporting big data process
Expand Down Expand Up @@ -510,7 +510,9 @@ def export_big_data_to_excel(dtable_uuid, table_id, view_id, username, name, tas
target_path = os.path.join(target_dir, excel_name)

images_target_dir = os.path.join(IMAGE_TMP_DIR, dtable_uuid, str(uuid.uuid4()))
image_param = {'num': 0, 'is_support': is_support_image, 'images_target_dir': images_target_dir}
image_param = {'num': 0, 'is_support': is_support_image, 'images_target_dir': images_target_dir,
'total_size': 0, 'max_size': int(images_max_size * 1024 * 1024) if images_max_size else EXPORT_IMAGE_MAX_SIZE,
'stop': False}

wb = openpyxl.Workbook(write_only=True)
ws = wb.create_sheet(sheet_name)
Expand Down Expand Up @@ -583,7 +585,7 @@ def export_big_data_to_excel(dtable_uuid, table_id, view_id, username, name, tas
pass


def export_app_table_page_to_excel(dtable_uuid, repo_id, table_id, username, app_name, page_name, filter_condition_groups, shown_column_keys, task_id, tasks_status_map, is_support_image=False):
def export_app_table_page_to_excel(dtable_uuid, repo_id, table_id, username, app_name, page_name, filter_condition_groups, shown_column_keys, task_id, tasks_status_map, is_support_image=False, images_max_size=None):
from dtable_events.dtable_io import dtable_io_logger

tasks_status_map[task_id] = {
Expand Down Expand Up @@ -633,7 +635,9 @@ def export_app_table_page_to_excel(dtable_uuid, repo_id, table_id, username, app
excel_name = app_name + '_' + page_name + '.xlsx'
target_path = os.path.join(target_dir, excel_name)
images_target_dir = os.path.join(IMAGE_TMP_DIR, dtable_uuid, str(uuid.uuid4()))
image_param = {'num': 0, 'is_support': is_support_image, 'images_target_dir': images_target_dir}
image_param = {'num': 0, 'is_support': is_support_image, 'images_target_dir': images_target_dir,
'total_size': 0, 'max_size': int(images_max_size * 1024 * 1024) if images_max_size else EXPORT_IMAGE_MAX_SIZE,
'stop': False}

wb = openpyxl.Workbook(write_only=True)
ws = wb.create_sheet(sheet_name)
Expand Down
28 changes: 21 additions & 7 deletions dtable_events/dtable_io/excel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@

EXPORT_IMAGE_LIMIT = 1000

EXPORT_IMAGE_MAX_SIZE = 100 * 1024 * 1024


EXCEL_IMPORT_DIR = '/tmp/dtable-io/'

Expand Down Expand Up @@ -1767,7 +1769,7 @@ def get_file_download_url(file_url, dtable_uuid, repo_id):
url = gen_file_get_url(token, asset_name)
return url

def add_image_to_excel(ws, cell_value, col_num, row_num, dtable_uuid, repo_id, image_num, images_target_dir, column, row_height):
def add_image_to_excel(ws, cell_value, col_num, row_num, dtable_uuid, repo_id, image_num, image_total_size, image_max_size, images_target_dir, column, row_height):
import requests
from openpyxl.drawing.image import Image
from PIL import Image as PILImage
Expand All @@ -1787,7 +1789,9 @@ def add_image_to_excel(ws, cell_value, col_num, row_num, dtable_uuid, repo_id, i
to_col_offset = -col_width * 7700
for image_url in images:
if image_num >= EXPORT_IMAGE_LIMIT:
return image_num
return {'num': image_num, 'total_size': image_total_size, 'stop': True}
if image_max_size and image_total_size >= image_max_size:
return {'num': image_num, 'total_size': image_total_size, 'stop': True}
real_image_url = urljoin(image_url, urlparse(image_url).path)

image_name = unquote(real_image_url.split('/')[-1].strip())
Expand All @@ -1800,6 +1804,7 @@ def add_image_to_excel(ws, cell_value, col_num, row_num, dtable_uuid, repo_id, i

response = requests.get(image_download_url)

@seafile-dev seafile-dev Aug 21, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Critical] 限制检查发生在完整下载和落盘之后

为什么重要：当前先用 requests.get() 将整张图片读入内存，并在临时目录写入文件、交给 Pillow/openpyxl 解析；直到这些操作完成后才在此处比较累计大小。因此单个超过上限的图片（或接近上限后的下一张大图）仍可占满 worker 内存和 /tmp，与本任务防止导出导致存储耗尽的目标不符；WebP 转 PNG 时还可能产生未计入 image_size 的更大临时文件。

建议修复：在创建临时文件前以流式下载执行上限校验：先校验可信的 Content-Length，并在每个 chunk 写入前按“已用额度 + 已写入字节”截断/删除；转换后应按实际待嵌入文件大小再次计数。还应为请求设置连接和读取超时。

image_content = response.content
image_size = len(image_content)

tmp_image_path = os.path.join(image_dir, image_name)
with open(tmp_image_path, 'wb') as f:
Expand Down Expand Up @@ -1838,11 +1843,15 @@ def add_image_to_excel(ws, cell_value, col_num, row_num, dtable_uuid, repo_id, i
to_anchor = AnchorMarker(col_num + 1, to_col_offset, row_num + 1, to_row_offset)
img.anchor = TwoCellAnchor('twoCell', from_anchor, to_anchor)

if image_max_size and image_total_size + image_size > image_max_size:
return {'num': image_num, 'total_size': image_total_size, 'stop': True}

ws.add_image(img)
if to_col_offset < 0:
from_col_offset += image_column_offset_transfer(row_height, img_width, image_height)
image_num += 1
return image_num
image_total_size += image_size
return {'num': image_num, 'total_size': image_total_size, 'stop': False}


def format_time(cell_data):
Expand Down Expand Up @@ -2020,11 +2029,16 @@ def _build_image_excel_cell(ws, cell_value, row_num, dtable_uuid, repo_id, image

c = WriteOnlyCell(ws)
image_num = image_param.get('num')
image_total_size = image_param.get('total_size', 0)
image_max_size = image_param.get('max_size')
is_stop = image_param.get('stop', False)
images_target_dir = image_param.get('images_target_dir')
if image_num < EXPORT_IMAGE_LIMIT:
num = add_image_to_excel(ws, cell_value, col_num, row_num, dtable_uuid, repo_id, image_num,
images_target_dir, column, row_height)
image_param['num'] = num
if not is_stop and image_num < EXPORT_IMAGE_LIMIT and (not image_max_size or image_total_size < image_max_size):
result = add_image_to_excel(ws, cell_value, col_num, row_num, dtable_uuid, repo_id, image_num,
image_total_size, image_max_size, images_target_dir, column, row_height)
image_param['num'] = result['num']
image_param['total_size'] = result['total_size']
image_param['stop'] = result['stop']
return c


Expand Down
21 changes: 17 additions & 4 deletions dtable_events/dtable_io/request_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,15 @@ def check_auth_token(req):
return True, None


def parse_images_max_size(value):
if not value:
return None
try:
return float(value) if float(value) > 0 else None
except (TypeError, ValueError):
return None


@app.route('/add-export-task', methods=['GET'])
def add_export_task():
from dtable_events.utils import parse_bool
Expand Down Expand Up @@ -1089,9 +1098,10 @@ def convert_view_to_excel():
name = context.get('name')
repo_id = context.get('repo_id')
is_support_image = to_python_boolean(context.get('is_support_image', 'false'))
images_max_size = parse_images_max_size(context.get('images_max_size'))

try:
task_id = task_manager.add_convert_view_to_excel_task(dtable_uuid, table_id, view_id, username, id_in_org, user_department_ids_map, permission, name, repo_id, is_support_image)
task_id = task_manager.add_convert_view_to_excel_task(dtable_uuid, table_id, view_id, username, id_in_org, user_department_ids_map, permission, name, repo_id, is_support_image, images_max_size)
except Exception as e:
dtable_io_logger.error(e)
return make_response((e, 500))
Expand All @@ -1117,9 +1127,10 @@ def convert_table_to_excel():
name = request.args.get('name')
repo_id = request.args.get('repo_id')
is_support_image = to_python_boolean(request.args.get('is_support_image', 'false'))
images_max_size = parse_images_max_size(request.args.get('images_max_size'))

try:
task_id = task_manager.add_convert_table_to_excel_task(dtable_uuid, table_id, username, name, repo_id, is_support_image)
task_id = task_manager.add_convert_table_to_excel_task(dtable_uuid, table_id, username, name, repo_id, is_support_image, images_max_size)
except Exception as e:
dtable_io_logger.error(e)
return make_response((e, 500))
Expand Down Expand Up @@ -1197,9 +1208,10 @@ def convert_big_data_view_to_excel():
name = data.get('name')
repo_id = data.get('repo_id')
is_support_image = to_python_boolean(data.get('is_support_image', 'false'))
images_max_size = parse_images_max_size(data.get('images_max_size'))

try:
task_id = big_data_task_manager.add_convert_big_data_view_to_excel_task(dtable_uuid, table_id, view_id, username, name, repo_id, is_support_image)
task_id = big_data_task_manager.add_convert_big_data_view_to_excel_task(dtable_uuid, table_id, view_id, username, name, repo_id, is_support_image, images_max_size)
except Exception as e:
dtable_io_logger.error(e)
return make_response((e, 500))
Expand Down Expand Up @@ -1431,9 +1443,10 @@ def convert_app_table_page_to_excel():
filter_condition_groups = json.loads(data.get('filter_condition_groups'))
shown_column_keys = json.loads(data.get('shown_column_keys'))
is_support_image = to_python_boolean(data.get('is_support_image', 'false'))
images_max_size = parse_images_max_size(data.get('images_max_size'))

try:
task_id = big_data_task_manager.add_convert_app_table_page_to_excel_task(dtable_uuid, repo_id, table_id, username, app_name, page_name, filter_condition_groups, shown_column_keys, is_support_image)
task_id = big_data_task_manager.add_convert_app_table_page_to_excel_task(dtable_uuid, repo_id, table_id, username, app_name, page_name, filter_condition_groups, shown_column_keys, is_support_image, images_max_size)
except Exception as e:
dtable_io_logger.error(e)
return make_response((e, 500))
Expand Down
8 changes: 4 additions & 4 deletions dtable_events/dtable_io/task_big_data_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,24 +94,24 @@ def add_update_big_excel_task(self, username, dtable_uuid, table_name, file_path
publish_metric(self.tasks_queue.qsize(), metric_name='big_data_io_task_queue_size', metric_help=BIG_DATA_TASK_MANAGER_METRIC_HELP)
return task_id

def add_convert_big_data_view_to_excel_task(self, dtable_uuid, table_id, view_id, username, name, repo_id, is_support_image):
def add_convert_big_data_view_to_excel_task(self, dtable_uuid, table_id, view_id, username, name, repo_id, is_support_image, images_max_size=None):
from dtable_events.dtable_io import convert_big_data_view_to_excel

task_id = str(uuid.uuid4())
task = (convert_big_data_view_to_excel,
(dtable_uuid, table_id, view_id, username, name, task_id, self.tasks_status_map, repo_id, is_support_image))
(dtable_uuid, table_id, view_id, username, name, task_id, self.tasks_status_map, repo_id, is_support_image, images_max_size))
self.tasks_queue.put(task_id)
self.tasks_map[task_id] = task
publish_metric(self.tasks_queue.qsize(), metric_name='big_data_io_task_queue_size', metric_help=BIG_DATA_TASK_MANAGER_METRIC_HELP)

return task_id

def add_convert_app_table_page_to_excel_task(self, dtable_uuid, repo_id, table_id, username, app_name, page_name, filter_condition_groups, shown_column_keys, is_support_image):
def add_convert_app_table_page_to_excel_task(self, dtable_uuid, repo_id, table_id, username, app_name, page_name, filter_condition_groups, shown_column_keys, is_support_image, images_max_size=None):
from dtable_events.dtable_io import convert_app_table_page_to_excel

task_id = str(uuid.uuid4())
task = (convert_app_table_page_to_excel,
(dtable_uuid, repo_id, table_id, username, app_name, page_name, filter_condition_groups, shown_column_keys, task_id, self.tasks_status_map, is_support_image))
(dtable_uuid, repo_id, table_id, username, app_name, page_name, filter_condition_groups, shown_column_keys, task_id, self.tasks_status_map, is_support_image, images_max_size))
self.tasks_queue.put(task_id)
self.tasks_map[task_id] = task
publish_metric(self.tasks_queue.qsize(), metric_name='big_data_io_task_queue_size', metric_help=BIG_DATA_TASK_MANAGER_METRIC_HELP)
Expand Down
8 changes: 4 additions & 4 deletions dtable_events/dtable_io/task_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -432,23 +432,23 @@ def add_force_sync_common_dataset_task(self, context):
return task_id, None

@log_function_call
def add_convert_view_to_excel_task(self, dtable_uuid, table_id, view_id, username, id_in_org, user_department_ids_map, permission, name, repo_id, is_support_image):
def add_convert_view_to_excel_task(self, dtable_uuid, table_id, view_id, username, id_in_org, user_department_ids_map, permission, name, repo_id, is_support_image, images_max_size=None):
from dtable_events.dtable_io import convert_view_to_excel

task_id = str(uuid.uuid4())
task = (convert_view_to_excel, (dtable_uuid, table_id, view_id, username, id_in_org, user_department_ids_map, permission, name, repo_id, is_support_image))
task = (convert_view_to_excel, (dtable_uuid, table_id, view_id, username, id_in_org, user_department_ids_map, permission, name, repo_id, is_support_image, images_max_size))
self.tasks_queue.put(task_id)
self.tasks_map[task_id] = task
publish_metric(self.tasks_queue.qsize(), 'io_task_queue_size', metric_help=TASK_MANAGER_METRIC_HELP)

return task_id

@log_function_call
def add_convert_table_to_excel_task(self, dtable_uuid, table_id, username, name, repo_id, is_support_image):
def add_convert_table_to_excel_task(self, dtable_uuid, table_id, username, name, repo_id, is_support_image, images_max_size=None):
from dtable_events.dtable_io import convert_table_to_excel

task_id = str(uuid.uuid4())
task = (convert_table_to_excel, (dtable_uuid, table_id, username, name, repo_id, is_support_image))
task = (convert_table_to_excel, (dtable_uuid, table_id, username, name, repo_id, is_support_image, images_max_size))
self.tasks_queue.put(task_id)
self.tasks_map[task_id] = task
publish_metric(self.tasks_queue.qsize(), 'io_task_queue_size', metric_help=TASK_MANAGER_METRIC_HELP)
Expand Down
Loading