Skip to content

Fix export excel bugs #906

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
freeplant merged 5 commits into from
Aug 21, 2026
Merged

Fix export excel bugs #906

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
538ecc2
export normal view query rows step by step 10000 -> 1000
AlexCXC Aug 20, 2026
0424ec6
convert view query rows by SQL query instead of dtable-db list-rows API
AlexCXC Aug 20, 2026
19b15bb
Recursive normalization filter
AlexCXC Aug 21, 2026
fa9c00c
fix grouped duration display format
AlexCXC Aug 21, 2026
f0d87af
Add more nested filters that depend on user information to unit tests.
AlexCXC Aug 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions dtable_events/dtable_io/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -972,6 +972,7 @@ def convert_view_to_excel(dtable_uuid, table_id, view_id, username, id_in_org, u
from dtable_events.dtable_io.excel import write_xls_with_type, TEMP_EXPORT_VIEW_DIR, IMAGE_TMP_DIR
from dtable_events.dtable_io.utils import get_related_nicknames_from_dtable, escape_sheet_name
from dtable_events.utils.dtable_db_api import DTableDBAPI
from dtable_events.utils.sql_generator import pre_filter_to_filter_term
import openpyxl

target_dir = TEMP_EXPORT_VIEW_DIR + dtable_uuid
Expand Down Expand Up @@ -1044,8 +1045,29 @@ def convert_view_to_excel(dtable_uuid, table_id, view_id, username, id_in_org, u
'user_department_ids_map': user_department_ids_map
}
dtable_db_api = DTableDBAPI(username, dtable_uuid, INNER_DTABLE_DB_URL, kwargs=kwargs)
try:
db_rows = get_export_view_rows_from_dtable_db(dtable_db_api, table_id=table_id, view_id=view_id)

filters = pre_filter_to_filter_term(
target_view.get('filters') or [],
username,
id_in_org,
(user_department_ids_map or {}).get('current_user_department_ids') or [],
(user_department_ids_map or {}).get('current_user_department_and_sub_ids') or [],
)
filter_conditions = {
'sorts': target_view.get('sorts'),
'filters': filters,
'filter_conjunction': target_view.get('filter_conjunction'),
}
selected_columns_map = {col.get('key'): col for col in cols_without_hidden}
for group_by in (target_view.get('groupbys') or []):
column_key = group_by.get('column_key')
if column_key and column_key not in selected_columns_map:
column = next((c for c in cols if c.get('key') == column_key), None)
if column:
selected_columns_map[column_key] = column
query_column_names = [col.get('name') for col in selected_columns_map.values()]
try:
db_rows = get_export_view_rows_from_dtable_db(dtable_db_api, table_name, cols, filter_conditions, query_column_names=query_column_names, server_only=True)
except Exception as e:
dtable_io_logger.error('get db rows. ERROR: {}'.format(e))
return
Expand Down
3 changes: 3 additions & 0 deletions dtable_events/dtable_io/excel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1896,6 +1896,9 @@ def handle_grouped_row(row, ws, cols_without_hidden, column_name_to_column, sub_
formula_value, number_format = parse_formula_number(first_cell_value, group_column.get('data'))
c = WriteOnlyCell(ws, value=formula_value)
c.number_format = number_format
elif group_column.get('type') == ColumnTypes.DURATION:
cell_value = format_duration(first_cell_value, group_column.get('data'))
c = WriteOnlyCell(ws, value=ILLEGAL_CHARACTERS_RE.sub('', cell_value))
else:
cell_value = cell_data2str(first_cell_value)
c = WriteOnlyCell(ws, value=ILLEGAL_CHARACTERS_RE.sub('', cell_value))
Expand Down
27 changes: 12 additions & 15 deletions dtable_events/dtable_io/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1535,24 +1535,21 @@ def get_rows_from_dtable_db(dtable_db_api, table_name, limit=50000):
return dtable_rows


def get_export_view_rows_from_dtable_db(dtable_db_api, table_name=None, table_id=None, view_name=None, view_id=None,
convert_link_id=True, convert_keys=True, convert_date=False):
def get_export_view_rows_from_dtable_db(dtable_db_api, table_name, columns, filter_conditions, query_column_names=None, server_only=True):
from dtable_events.utils.sql_generator import filter2sql

rows = []
start, limit = 0, 10000
while True:
rows_rsp = dtable_db_api.list_rows(
table_name=table_name,
table_id=table_id,
view_name=view_name,
view_id=view_id,
start=start,
limit=limit,
convert_link_id=convert_link_id,
convert_keys=convert_keys,
convert_date=convert_date
)
rows.extend(rows_rsp['rows'])
if len(rows_rsp['rows']) < limit:
filter_conditions['start'] = start
filter_conditions['limit'] = limit
sql = filter2sql(table_name, columns, filter_conditions, by_group=False)

@seafile-dev seafile-dev Aug 20, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Critical] SQL 未排除归档行

为什么重要：
设计要求普通视图只读取非归档存储，但这里生成的 SQL 仅包含视图过滤、排序和 LIMIT。归档行只要满足视图过滤就会被导出，导致普通视图导出包含不应出现的数据。

建议修复：在生成导出 SQL 时无条件追加非归档谓词（并正确合并已有 WHERE），或给 filter2sql 增加明确的 normal-view/archived 开关，并补充归档行回归用例。

if query_column_names:
column_names_str = ', '.join('`%s`' % name for name in query_column_names)
sql = sql.replace('*', column_names_str, 1)
step_rows, _ = dtable_db_api.query(sql, convert=True, server_only=server_only)
rows.extend(step_rows)
if len(step_rows) < limit:
break
start += limit
return rows
Expand Down
10 changes: 9 additions & 1 deletion dtable_events/tests/sql/column_reference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,15 @@
{'key': 'A47g', 'type': 'auto-number', 'name': 'AutoNo', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'format': '0000', 'max_used_auto_number': 906, 'digits': 4, 'prefix_type': None, 'prefix': None}, 'permission_type': '', 'permitted_users': []},
{'key': 'G0yz', 'type': 'checkbox', 'name': 'CB', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'enable_fill_default_value': False, 'default_value': False}, 'permission_type': '', 'permitted_users': []},
{'key': '6SJV', 'type': 'duration', 'name': 'Du', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'format': 'duration', 'duration_format': 'h:mm:ss'}, 'permission_type': '', 'permitted_users': []},
{'key': '51r0', 'type': 'number', 'name': 'Num', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'format': 'number', 'precision': 2, 'enable_precision': False, 'enable_fill_default_value': False, 'decimal': 'dot', 'thousands': 'no'}, 'permission_type': '', 'permitted_users': []}
{'key': '51r0', 'type': 'number', 'name': 'Num', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'format': 'number', 'precision': 2, 'enable_precision': False, 'enable_fill_default_value': False, 'decimal': 'dot', 'thousands': 'no'}, 'permission_type': '', 'permitted_users': []},
{'key': 'F000', 'type': 'formula', 'name': 'FNum', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'result_type': 'number'}, 'permission_type': '', 'permitted_users': []},
{'key': 'F001', 'type': 'formula', 'name': 'FStr', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'result_type': 'string'}, 'permission_type': '', 'permitted_users': []},
{'key': 'F002', 'type': 'formula', 'name': 'FDate', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'result_type': 'date', 'format': 'YYYY-MM-DD'}, 'permission_type': '', 'permitted_users': []},
{'key': 'F003', 'type': 'formula', 'name': 'FBool', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'result_type': 'bool'}, 'permission_type': '', 'permitted_users': []},
{'key': 'F004', 'type': 'formula', 'name': 'FArr', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'result_type': 'array', 'array_type': 'text'}, 'permission_type': '', 'permitted_users': []},
{'key': 'L001', 'type': 'link', 'name': 'LNum', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'array_type': 'number', 'result_type': 'array'}, 'permission_type': '', 'permitted_users': []},
{'key': 'D001', 'type': 'department-single-select', 'name': 'Dept', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'options': [{'id': 1, 'name': 'Dept1'}, {'id': 2, 'name': 'Dept2'}]}, 'permission_type': '', 'permitted_users': []},
{'key': 'G001', 'type': 'geolocation', 'name': 'Geo', 'editable': True, 'width': 200, 'resizable': True, 'draggable': True, 'data': {'format': 'lng_lat'}, 'permission_type': '', 'permitted_users': []}
]

LINK_COLUMN = {"key":"G5J9","type":"link","name":"Link","editable":True,"width":200,"resizable":True,"draggable":True,"data":{"display_column_key":"0000","table_id":"0000","other_table_id":"kzvB","is_internal_link":True,"is_multiple":True,"is_row_from_view":False,"other_view_id":"","link_id":"55Sl","array_type":"text","array_data":None,"result_type":"array"},"permission_type":"","permitted_users":[],"edit_metadata_permission_type":"","edit_metadata_permitted_users":[],"description":None}
Expand Down
118 changes: 118 additions & 0 deletions dtable_events/tests/sql/sql_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from sql.column_reference import TEST_COLUMNS, TABLES, LINK_COLUMN
from sql.test_reference import TEST_CONDITIONS, TEST_CONDITIONS_LINK
from dtable_events import filter2sql, linkRecords2sql
from dtable_events.utils.sql_generator import pre_filter_to_filter_term

class SqlTest(unittest.TestCase):

Expand Down Expand Up @@ -52,5 +53,122 @@ def test_equal(self):



def test_user_filter_normalization(self):

def to_sql(filters):
normalized = pre_filter_to_filter_term(filters, 'me@x.com', 'admin-1', [1, 2], [1, 2, 3])
return self._toSql({'filters': normalized, 'filter_conjunction': 'And'})

# collaborator include_me appends current user email
self.assertEqual(
to_sql([{'column_name': 'Colla', 'filter_predicate': 'include_me', 'filter_term': ['a@x.com']}]),
"SELECT * FROM `Table1` WHERE (`Colla` in ('a@x.com', 'me@x.com')) LIMIT 0, 100",
)
# text is_current_user_ID replaced with id_in_org
self.assertEqual(
to_sql([{'column_name': '名称', 'filter_predicate': 'is_current_user_ID', 'filter_term': ''}]),
"SELECT * FROM `Table1` WHERE (`名称` = 'admin-1') LIMIT 0, 100",
)
# department current_user_department / current_user_department_and_sub
self.assertEqual(
to_sql([{'column_name': 'Dept', 'filter_predicate': 'is', 'filter_term': 'current_user_department'}]),
"SELECT * FROM `Table1` WHERE (`Dept` IN (1, 2)) LIMIT 0, 100",
)
self.assertEqual(
to_sql([{'column_name': 'Dept', 'filter_predicate': 'is_not', 'filter_term': 'current_user_department_and_sub'}]),
"SELECT * FROM `Table1` WHERE (`Dept` NOT IN (1, 2, 3)) LIMIT 0, 100",
)
# nested filter group include_me
self.assertEqual(
to_sql([{'filters': [{'column_name': 'Colla', 'filter_predicate': 'include_me', 'filter_term': ['b@x.com']}], 'filter_conjunction': 'And'}]),
"SELECT * FROM `Table1` WHERE ((`Colla` in ('b@x.com', 'me@x.com'))) LIMIT 0, 100",
)
# deeply nested filter group (3 levels) include_me must still be normalized
self.assertEqual(
to_sql([
{
'filters': [
{
'filters': [
{'column_name': 'Colla', 'filter_predicate': 'include_me', 'filter_term': ['c@x.com']},
],
'filter_conjunction': 'And',
},
],
'filter_conjunction': 'And',
},
]),
"SELECT * FROM `Table1` WHERE (((`Colla` in ('c@x.com', 'me@x.com')))) LIMIT 0, 100",
)
# deeply nested filter group (3 levels) current_user_department must still be normalized

@seafile-dev seafile-dev Aug 21, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Warning] 深层用户条件测试覆盖不足

为什么重要：递归修复后的三层嵌套断言只覆盖了“包含我”和“当前用户所在部门”。“是当前用户 ID”及“当前用户所在部门 / 含子部门”仍只在顶层验证，未来这两条分支在递归路径回归时无法被测试拦截。

建议修复：在这里补充三层嵌套的 is_current_user_ID、current_user_department_and_sub（以及列表取值形式如适用）SQL 断言，确认每种当前用户条件都会被递归归一化。

self.assertEqual(
to_sql([
{
'filters': [
{
'filters': [
{'column_name': 'Dept', 'filter_predicate': 'is', 'filter_term': 'current_user_department'},
],
'filter_conjunction': 'And',
},
],
'filter_conjunction': 'And',
},
]),
"SELECT * FROM `Table1` WHERE (((`Dept` IN (1, 2)))) LIMIT 0, 100",
)
# deeply nested filter group (3 levels) is_current_user_ID must still be normalized
self.assertEqual(
to_sql([
{
'filters': [
{
'filters': [
{'column_name': '名称', 'filter_predicate': 'is_current_user_ID', 'filter_term': ''},
],
'filter_conjunction': 'And',
},
],
'filter_conjunction': 'And',
},
]),
"SELECT * FROM `Table1` WHERE (((`名称` = 'admin-1'))) LIMIT 0, 100",
)
# deeply nested filter group (3 levels) current_user_department_and_sub must still be normalized
self.assertEqual(
to_sql([
{
'filters': [
{
'filters': [
{'column_name': 'Dept', 'filter_predicate': 'is', 'filter_term': 'current_user_department_and_sub'},
],
'filter_conjunction': 'And',
},
],
'filter_conjunction': 'And',
},
]),
"SELECT * FROM `Table1` WHERE (((`Dept` IN (1, 2, 3)))) LIMIT 0, 100",
)
# deeply nested filter group (3 levels) list-valued department filter must still be normalized
self.assertEqual(
to_sql([
{
'filters': [
{
'filters': [
{'column_name': 'Dept', 'filter_predicate': 'is_any_of', 'filter_term': ['current_user_department_and_sub', 999]},
],
'filter_conjunction': 'And',
},
],
'filter_conjunction': 'And',
},
]),
"SELECT * FROM `Table1` WHERE (((`Dept` IN (1, 2, 3, 999)))) LIMIT 0, 100",
)


if __name__ == '__main__':
unittest.main()
133 changes: 133 additions & 0 deletions dtable_events/tests/sql/test_reference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
import time
from datetime import timedelta, datetime

from dtable_events.utils.sql_generator import ColumnFilterInvalidError

# set timezone
os.environ['TZ'] = 'UTC'
time.tzset() # 在Unix-like系统上生效
Expand Down Expand Up @@ -693,6 +695,137 @@ def get_expected_sql_for_modifier(filter_modifier, column_name):
"by_group": True,
"expected_error": ValueError
},

# Formula column, result_type = number
{
"filter_conditions": {
"filters": [
{'column_name': 'FNum', 'filter_predicate': 'equal', 'filter_term': 5}
],
"filter_predicate": 'And',
"sorts": [],
},
"expected_sql": "SELECT * FROM `Table1` WHERE (`FNum` = 5) LIMIT 0, 100",
"by_group": False,
},
# Formula column, result_type = string
{
"filter_conditions": {
"filters": [
{'column_name': 'FStr', 'filter_predicate': 'contains', 'filter_term': 'abc'}
],
"filter_predicate": 'And',
"sorts": [],
},
"expected_sql": "SELECT * FROM `Table1` WHERE (`FStr` ilike '%abc%') LIMIT 0, 100",
"by_group": False,
},
# Formula column, result_type = date
{
"filter_conditions": {
"filters": [
{'column_name': 'FDate', 'filter_predicate': 'is', 'filter_term': '2021-12-20', 'filter_term_modifier': 'exact_date'}
],
"filter_predicate": 'And',
"sorts": [],
},
"expected_sql": "SELECT * FROM `Table1` WHERE ((`FDate` >= '2021-12-20' and `FDate` < '2021-12-21')) LIMIT 0, 100",
"by_group": False,
},
# Formula column, result_type = bool
{
"filter_conditions": {
"filters": [
{'column_name': 'FBool', 'filter_predicate': 'is', 'filter_term': 'true'}
],
"filter_predicate": 'And',
"sorts": [],
},
"expected_sql": "SELECT * FROM `Table1` WHERE (`FBool` = true) LIMIT 0, 100",
"by_group": False,
},
# Formula column, result_type = array (array_type text)
{
"filter_conditions": {
"filters": [
{'column_name': 'FArr', 'filter_predicate': 'contains', 'filter_term': 'abc'}
],
"filter_predicate": 'And',
"sorts": [],
},
"expected_sql": "SELECT * FROM `Table1` WHERE (`FArr` ilike '%abc%') LIMIT 0, 100",
"by_group": False,
},
# Link column, array_type = number
{
"filter_conditions": {
"filters": [
{'column_name': 'LNum', 'filter_predicate': 'equal', 'filter_term': 5}
],
"filter_predicate": 'And',
"sorts": [],
},
"expected_sql": "SELECT * FROM `Table1` WHERE (`LNum` = 5) LIMIT 0, 100",
"by_group": False,
},
# Department single select
{
"filter_conditions": {
"filters": [
{'column_name': 'Dept', 'filter_predicate': 'is', 'filter_term': 1}
],
"filter_predicate": 'And',
"sorts": [],
},
"expected_sql": "SELECT * FROM `Table1` WHERE (`Dept` = 1) LIMIT 0, 100",
"by_group": False,
},
{
"filter_conditions": {
"filters": [
{'column_name': 'Dept', 'filter_predicate': 'is_any_of', 'filter_term': [1, 2]}
],
"filter_predicate": 'And',
"sorts": [],
},
"expected_sql": "SELECT * FROM `Table1` WHERE (`Dept` IN (1, 2)) LIMIT 0, 100",
"by_group": False,
},
# Geolocation
{
"filter_conditions": {
"filters": [
{'column_name': 'Geo', 'filter_predicate': 'is_empty'}
],
"filter_predicate": 'And',
"sorts": [],
},
"expected_sql": "SELECT * FROM `Table1` WHERE (`Geo` is null) LIMIT 0, 100",
"by_group": False,
},
{
"filter_conditions": {
"filters": [
{'column_name': 'Geo', 'filter_predicate': 'is_not_empty'}
],
"filter_predicate": 'And',
"sorts": [],
},
"expected_sql": "SELECT * FROM `Table1` WHERE (`Geo` is not null) LIMIT 0, 100",
"by_group": False,
},
# Unsupported predicate raises ColumnFilterInvalidError
{
"filter_conditions": {
"filters": [
{'column_name': 'Num', 'filter_predicate': 'has_any_of', 'filter_term': [1]}
],
"filter_predicate": 'And',
"sorts": [],
},
"by_group": False,
"expected_error": ColumnFilterInvalidError,
},
]

TEST_CONDITIONS_LINK = [
Expand Down
Loading
Loading