From bc4685c8654c3d04416f1740f2637ca9e9b43d26 Mon Sep 17 00:00:00 2001 From: sonika-shah <58761340+sonika-shah@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:22:26 +0530 Subject: [PATCH 1/3] Fixes #31857: preserve table constraints on recursive CSV import createTableEntity fetched the existing table without tableConstraints/ tablePartition, so the subsequent createOrUpdate persisted them as null and the recursive CSV import silently dropped every table's PK/UNIQUE/FK. Load those fields so they carry through unchanged. Adds DatabaseServiceResourceIT.test_importExportRecursive_preservesTableConstraints. --- .../it/tests/DatabaseServiceResourceIT.java | 82 +++++++++++++++++++ .../java/org/openmetadata/csv/EntityCsv.java | 9 +- 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java index 7f50e22c06ba..03e6d3e19b77 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java @@ -53,6 +53,7 @@ import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.ColumnDataType; import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.type.TableConstraint; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.csv.CsvImportResult; import org.openmetadata.sdk.models.ListParams; @@ -788,6 +789,87 @@ void test_importExportRecursive_withColumnTagsAndGlossaryTerms(TestNamespace ns) "Street column should STILL have glossary term (not removed)"); } + @Test + void test_importExportRecursive_preservesTableConstraints(TestNamespace ns) + throws IOException, InterruptedException { + String serviceName = ns.prefix("import_export_recursive_constraints_service"); + DatabaseService service = createEntity(createMinimalRequest(ns).withName(serviceName)); + + Database database = + SdkClients.adminClient() + .databases() + .create( + new CreateDatabase() + .withName(ns.prefix("db1")) + .withService(service.getFullyQualifiedName())); + + DatabaseSchema schema = + SdkClients.adminClient() + .databaseSchemas() + .create( + new CreateDatabaseSchema() + .withName(ns.prefix("schema1")) + .withDatabase(database.getFullyQualifiedName())); + + List constraints = + List.of( + new TableConstraint() + .withConstraintType(TableConstraint.ConstraintType.PRIMARY_KEY) + .withColumns(List.of("id")), + new TableConstraint() + .withConstraintType(TableConstraint.ConstraintType.UNIQUE) + .withColumns(List.of("email"))); + + Table table = + SdkClients.adminClient() + .tables() + .create( + new CreateTable() + .withName(ns.prefix("constrained_table")) + .withDatabaseSchema(schema.getFullyQualifiedName()) + .withColumns( + List.of( + new Column().withName("id").withDataType(ColumnDataType.BIGINT), + new Column() + .withName("email") + .withDataType(ColumnDataType.VARCHAR) + .withDataLength(255))) + .withTableConstraints(constraints)); + + assertEquals( + 2, table.getTableConstraints().size(), "Table should start with 2 table constraints"); + + // Recursive export then re-import unchanged: this is the whole-tree path the UI uses when + // importing at service/database/schema level. The recursive CSV has no column for table + // constraints, so a round trip must not drop them. + String exportedCsv = exportCsvRecursive(service.getFullyQualifiedName()); + assertNotNull(exportedCsv); + + CsvImportResult result = + importCsvRecursive(service.getFullyQualifiedName(), exportedCsv, false); + assertEquals(ApiStatus.SUCCESS, result.getStatus()); + + Table reloaded = + SdkClients.adminClient() + .tables() + .getByName(table.getFullyQualifiedName(), "tableConstraints"); + assertNotNull( + reloaded.getTableConstraints(), + "Table constraints must survive a recursive CSV round trip"); + assertEquals( + 2, + reloaded.getTableConstraints().size(), + "Both PRIMARY_KEY and UNIQUE constraints must be preserved after a recursive import"); + assertTrue( + reloaded.getTableConstraints().stream() + .anyMatch(c -> c.getConstraintType() == TableConstraint.ConstraintType.PRIMARY_KEY), + "PRIMARY_KEY constraint must be preserved after a recursive import"); + assertTrue( + reloaded.getTableConstraints().stream() + .anyMatch(c -> c.getConstraintType() == TableConstraint.ConstraintType.UNIQUE), + "UNIQUE constraint must be preserved after a recursive import"); + } + private String addColumnTags(String csvLine, String tagFQN) { String[] parts = splitCsvRow(csvLine); if (parts.length >= 5) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index e873fe616f65..2ee7bee321d4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -1743,11 +1743,16 @@ protected void createTableEntity(CSVPrinter printer, CSVRecord csvRecord, String .withId(UUID.randomUUID()); } } else { - // Dry Run = false, True Run: Use dependency resolution helper + // Dry Run = false, True Run: Use dependency resolution helper. + // Fetch tableConstraints/tablePartition too: the recursive CSV has no column for them, so + // they must be carried through the createOrUpdate below or they would be dropped. try { table = getEntityWithDependencyResolution( - TABLE, tableFqn, "owners,tags,domains,extension", Include.NON_DELETED); + TABLE, + tableFqn, + "owners,tags,domains,extension,tableConstraints,tablePartition", + Include.NON_DELETED); } catch (EntityNotFoundException ex) { // Table not found, create a new one LOG.warn("Table not found: {}, it will be created with Import.", tableFqn); From 7dd2bc052f68e4b24506d4b8a084a520386bfdac Mon Sep 17 00:00:00 2001 From: sonika-shah <58761340+sonika-shah@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:49:19 +0530 Subject: [PATCH 2/3] test: cover FOREIGN_KEY (ER) in recursive CSV constraint round trip Adds a referenced table and a FOREIGN_KEY constraint (with referredColumns) so the recursive round trip also asserts the referenced-table linkage survives, not just PRIMARY_KEY/UNIQUE. --- .../it/tests/DatabaseServiceResourceIT.java | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java index 03e6d3e19b77..188df0400c2c 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java @@ -811,6 +811,18 @@ void test_importExportRecursive_preservesTableConstraints(TestNamespace ns) .withName(ns.prefix("schema1")) .withDatabase(database.getFullyQualifiedName())); + // Referenced table for the FOREIGN_KEY (exercises the table-to-table RELATED_TO edge path). + Table refTable = + SdkClients.adminClient() + .tables() + .create( + new CreateTable() + .withName(ns.prefix("ref_table")) + .withDatabaseSchema(schema.getFullyQualifiedName()) + .withColumns( + List.of(new Column().withName("ref_id").withDataType(ColumnDataType.BIGINT)))); + String refIdFqn = refTable.getFullyQualifiedName() + ".ref_id"; + List constraints = List.of( new TableConstraint() @@ -818,7 +830,11 @@ void test_importExportRecursive_preservesTableConstraints(TestNamespace ns) .withColumns(List.of("id")), new TableConstraint() .withConstraintType(TableConstraint.ConstraintType.UNIQUE) - .withColumns(List.of("email"))); + .withColumns(List.of("email")), + new TableConstraint() + .withConstraintType(TableConstraint.ConstraintType.FOREIGN_KEY) + .withColumns(List.of("ref_fk")) + .withReferredColumns(List.of(refIdFqn))); Table table = SdkClients.adminClient() @@ -833,11 +849,12 @@ void test_importExportRecursive_preservesTableConstraints(TestNamespace ns) new Column() .withName("email") .withDataType(ColumnDataType.VARCHAR) - .withDataLength(255))) + .withDataLength(255), + new Column().withName("ref_fk").withDataType(ColumnDataType.BIGINT))) .withTableConstraints(constraints)); assertEquals( - 2, table.getTableConstraints().size(), "Table should start with 2 table constraints"); + 3, table.getTableConstraints().size(), "Table should start with 3 table constraints"); // Recursive export then re-import unchanged: this is the whole-tree path the UI uses when // importing at service/database/schema level. The recursive CSV has no column for table @@ -857,9 +874,9 @@ void test_importExportRecursive_preservesTableConstraints(TestNamespace ns) reloaded.getTableConstraints(), "Table constraints must survive a recursive CSV round trip"); assertEquals( - 2, + 3, reloaded.getTableConstraints().size(), - "Both PRIMARY_KEY and UNIQUE constraints must be preserved after a recursive import"); + "PRIMARY_KEY, UNIQUE and FOREIGN_KEY constraints must all be preserved after recursive import"); assertTrue( reloaded.getTableConstraints().stream() .anyMatch(c -> c.getConstraintType() == TableConstraint.ConstraintType.PRIMARY_KEY), @@ -868,6 +885,17 @@ void test_importExportRecursive_preservesTableConstraints(TestNamespace ns) reloaded.getTableConstraints().stream() .anyMatch(c -> c.getConstraintType() == TableConstraint.ConstraintType.UNIQUE), "UNIQUE constraint must be preserved after a recursive import"); + // FOREIGN_KEY carries the referenced-table linkage (referredColumns); it must survive intact. + TableConstraint fk = + reloaded.getTableConstraints().stream() + .filter(c -> c.getConstraintType() == TableConstraint.ConstraintType.FOREIGN_KEY) + .findFirst() + .orElse(null); + assertNotNull(fk, "FOREIGN_KEY constraint must be preserved after a recursive import"); + assertEquals( + List.of(refIdFqn), + fk.getReferredColumns(), + "FOREIGN_KEY referredColumns (referenced-table linkage) must be preserved"); } private String addColumnTags(String csvLine, String tagFQN) { From 940093e09f88135b8b2e4cbb5a5480d4ec3f1032 Mon Sep 17 00:00:00 2001 From: sonika-shah <58761340+sonika-shah@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:04:59 +0530 Subject: [PATCH 3/3] Fixes #31858: preserve column custom properties on recursive CSV import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit createTableEntity fetched the table with "extension" but not "columns"; TableRepository.setFields only hydrates column.extension when both fields are requested, so the recursive table-row createOrUpdate persisted columns with null extension and dropped column custom properties before the column rows patched back. Add "columns" to the fetch so column extension is hydrated and carried through. Adds DatabaseServiceResourceIT.test_importExportRecursive_preservesColumnCustomProperties. Verified: 3 recursive round-trip ITs pass (constraints+FK, column custom properties, and the pre-existing column tags/glossary test) — Tests run: 3, Failures: 0, Errors: 0. --- .../it/tests/DatabaseServiceResourceIT.java | 92 ++++++++++++++++++- .../java/org/openmetadata/csv/EntityCsv.java | 7 +- 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java index 188df0400c2c..9906bd7d9b2c 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DatabaseServiceResourceIT.java @@ -13,6 +13,7 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.util.List; +import java.util.Map; import java.util.UUID; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; @@ -820,7 +821,8 @@ void test_importExportRecursive_preservesTableConstraints(TestNamespace ns) .withName(ns.prefix("ref_table")) .withDatabaseSchema(schema.getFullyQualifiedName()) .withColumns( - List.of(new Column().withName("ref_id").withDataType(ColumnDataType.BIGINT)))); + List.of( + new Column().withName("ref_id").withDataType(ColumnDataType.BIGINT)))); String refIdFqn = refTable.getFullyQualifiedName() + ".ref_id"; List constraints = @@ -898,6 +900,94 @@ void test_importExportRecursive_preservesTableConstraints(TestNamespace ns) "FOREIGN_KEY referredColumns (referenced-table linkage) must be preserved"); } + @Test + void test_importExportRecursive_preservesColumnCustomProperties(TestNamespace ns) + throws IOException, InterruptedException { + String serviceName = ns.prefix("import_export_recursive_col_ext_service"); + DatabaseService service = createEntity(createMinimalRequest(ns).withName(serviceName)); + + Database database = + SdkClients.adminClient() + .databases() + .create( + new CreateDatabase() + .withName(ns.prefix("db1")) + .withService(service.getFullyQualifiedName())); + + DatabaseSchema schema = + SdkClients.adminClient() + .databaseSchemas() + .create( + new CreateDatabaseSchema() + .withName(ns.prefix("schema1")) + .withDatabase(database.getFullyQualifiedName())); + + // Columns carry free-form custom properties (column.extension). + Column idColumn = + new Column() + .withName("id") + .withDataType(ColumnDataType.BIGINT) + .withExtension(Map.of("colGovOwner", "id-col@example.com")); + Column emailColumn = + new Column() + .withName("email") + .withDataType(ColumnDataType.VARCHAR) + .withDataLength(255) + .withExtension(Map.of("colGovOwner", "email-col@example.com")); + + Table table = + SdkClients.adminClient() + .tables() + .create( + new CreateTable() + .withName(ns.prefix("col_ext_table")) + .withDatabaseSchema(schema.getFullyQualifiedName()) + .withColumns(List.of(idColumn, emailColumn))); + + // Precondition: column custom properties persisted on create. + Table created = + SdkClients.adminClient() + .tables() + .getByName(table.getFullyQualifiedName(), "columns,extension"); + assertNotNull( + columnByName(created, "id").getExtension(), + "Column custom properties should persist on create"); + + // Recursive export then re-import unchanged (the whole-tree path the UI uses at + // service/database/schema level). The recursive CSV has no column for column custom + // properties, so a round trip must not drop them. + String exportedCsv = exportCsvRecursive(service.getFullyQualifiedName()); + assertNotNull(exportedCsv); + + CsvImportResult result = + importCsvRecursive(service.getFullyQualifiedName(), exportedCsv, false); + assertEquals(ApiStatus.SUCCESS, result.getStatus()); + + Table reloaded = + SdkClients.adminClient() + .tables() + .getByName(table.getFullyQualifiedName(), "columns,extension"); + assertNotNull( + columnByName(reloaded, "id").getExtension(), + "Column 'id' custom properties must survive a recursive CSV round trip"); + assertTrue( + columnByName(reloaded, "id").getExtension().toString().contains("id-col@example.com"), + "Column 'id' custom property value must be preserved after a recursive import"); + assertNotNull( + columnByName(reloaded, "email").getExtension(), + "Column 'email' custom properties must survive a recursive CSV round trip"); + assertTrue( + columnByName(reloaded, "email").getExtension().toString().contains("email-col@example.com"), + "Column 'email' custom property value must be preserved after a recursive import"); + } + + private Column columnByName(Table table, String name) { + return table.getColumns().stream() + .filter(c -> name.equals(c.getName())) + .findFirst() + .orElseThrow(() -> new AssertionError("Column not found: " + name)); + } + private String addColumnTags(String csvLine, String tagFQN) { String[] parts = splitCsvRow(csvLine); if (parts.length >= 5) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 2ee7bee321d4..656978ce3cd3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -1744,14 +1744,15 @@ protected void createTableEntity(CSVPrinter printer, CSVRecord csvRecord, String } } else { // Dry Run = false, True Run: Use dependency resolution helper. - // Fetch tableConstraints/tablePartition too: the recursive CSV has no column for them, so - // they must be carried through the createOrUpdate below or they would be dropped. + // Fetch tableConstraints/tablePartition (the recursive CSV has no column for them) and + // columns (column extension is only hydrated when both "columns" and "extension" are + // requested) so the createOrUpdate below carries them through instead of dropping them. try { table = getEntityWithDependencyResolution( TABLE, tableFqn, - "owners,tags,domains,extension,tableConstraints,tablePartition", + "owners,tags,domains,extension,tableConstraints,tablePartition,columns", Include.NON_DELETED); } catch (EntityNotFoundException ex) { // Table not found, create a new one