From b393ef08128d75839b80ed214b2ede903add9266 Mon Sep 17 00:00:00 2001 From: Sun Cao Date: Fri, 13 Nov 2026 06:58:36 +0000 Subject: [PATCH] Relay Kernel's report on a version, timestamp, or log it cannot read A version, timestamp, or log that Delta Kernel rejects, such as a version after the latest, a timestamp before the first commit, or a missing log file, is answered with 400 BAD_REQUEST and Kernel's message, instead of the server's own wording for the cases it checked first. A version after the latest is answered with Kernel's message without asking Kernel: for a version far ahead, such as 999999, Kernel searches back through the log for a checkpoint, which takes minutes on object storage. Co-authored-by: Isaac Signed-off-by: Sun Cao --- .../asset/table/delta/DeltaKernel.java | 11 ++-- .../asset/table/delta/DeltaSnapshots.java | 54 +++++++++---------- .../java/io/opensharing/http/ApiFailure.java | 4 ++ .../java/io/opensharing/http/ErrorCodes.java | 1 + .../io/opensharing/http/ApiFailureTest.java | 12 +++++ .../protocol/ProtocolApiSupport.java | 7 +++ .../ProtocolTableMetadataIntegrationTest.java | 4 +- .../protocol/ProtocolTableMetadataTest.java | 8 ++- .../ProtocolTableQueryIntegrationTest.java | 9 ++-- .../protocol/ProtocolTableQueryTest.java | 6 ++- .../ProtocolTableVersionIntegrationTest.java | 6 +-- .../protocol/ProtocolTableVersionTest.java | 3 +- .../SparkSharedTableQueryIntegrationTest.java | 2 +- 13 files changed, 76 insertions(+), 51 deletions(-) diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaKernel.java b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaKernel.java index f76e2e5c4..3bfee5486 100644 --- a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaKernel.java +++ b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaKernel.java @@ -49,14 +49,9 @@ public long getVersion(ResolvedAsset table, Instant startingTimestamp, AuthConte static long versionAtOrAfter(TableImpl table, Engine engine, Instant startingTimestamp) { Snapshot latest = latestSnapshot(table, engine); - if (startingTimestamp == null) { - return latest.getVersion(); - } - if (startingTimestamp.toEpochMilli() > latest.getTimestamp(engine)) { - throw ApiException.invalidParameter( - "startingTimestamp is after the latest table version timestamp"); - } - return table.getVersionAtOrAfterTimestamp(engine, startingTimestamp.toEpochMilli()); + return startingTimestamp == null + ? latest.getVersion() + : table.getVersionAtOrAfterTimestamp(engine, startingTimestamp.toEpochMilli()); } /** diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaSnapshots.java b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaSnapshots.java index 2054b91d7..4dbeb8f9b 100644 --- a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaSnapshots.java +++ b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaSnapshots.java @@ -5,6 +5,7 @@ import io.delta.kernel.internal.SnapshotImpl; import io.delta.kernel.internal.checksum.CRCInfo; import io.opensharing.http.ApiException; +import io.opensharing.http.ErrorCodes; import java.time.Instant; /** Opens a Kernel snapshot for a version, timestamp, or the latest log, and reads its checksum. */ @@ -14,30 +15,31 @@ private DeltaSnapshots() {} static Snapshot open(DeltaKernel.Session session, Long version, Instant timestamp) { Snapshot latest = DeltaKernel.latestSnapshot(session.table(), session.engine()); - if (version != null) { - if (version > latest.getVersion()) { - throw ApiException.invalidParameter( - "version " + version + " is after the latest table version " + latest.getVersion()); - } - try { - return session.table().getSnapshotAsOfVersion(session.engine(), version); - } catch (KernelException | IllegalArgumentException unavailable) { - throw unavailable(unavailable, "version " + version + " is no longer available"); - } + if (version == null && timestamp == null) { + return latest; } - if (timestamp != null) { - if (timestamp.toEpochMilli() > latest.getTimestamp(session.engine())) { - throw ApiException.invalidParameter( - "timestamp is after the latest table version timestamp"); - } - try { - return session.table().getSnapshotAsOfTimestamp(session.engine(), timestamp.toEpochMilli()); - } catch (KernelException | IllegalArgumentException unavailable) { - throw unavailable( - unavailable, "timestamp is before the earliest available table version timestamp"); - } + if (version != null && version > latest.getVersion()) { + // Kernel's message, table path included, without the search back through the log that it + // makes for a version this far ahead, which takes minutes on object storage. + throw new ApiException( + 400, + ErrorCodes.BAD_REQUEST, + session.table().getPath(session.engine()) + + ": Cannot load table version " + + version + + " as it does not exist. The latest available version is " + + latest.getVersion() + + "."); + } + try { + return version != null + ? session.table().getSnapshotAsOfVersion(session.engine(), version) + : session.table().getSnapshotAsOfTimestamp(session.engine(), timestamp.toEpochMilli()); + } catch (KernelException unavailable) { + throw DeltaKernel.unsupportedProtocol(unavailable) + ? DeltaKernel.unreadableTable(unavailable) + : unavailable; } - return latest; } /** @@ -50,12 +52,4 @@ static CRCInfo crcFor(SnapshotImpl snapshot) { .filter(crc -> crc.getVersion() == snapshot.getVersion()) .orElse(null); } - - /** An earlier snapshot can need a protocol Kernel cannot read even when the latest does not. */ - private static ApiException unavailable(RuntimeException failed, String message) { - if (failed instanceof KernelException kernel && DeltaKernel.unsupportedProtocol(kernel)) { - return DeltaKernel.unreadableTable(kernel); - } - return ApiException.invalidParameter(message); - } } diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/http/ApiFailure.java b/server/opensharing-server-core/src/main/java/io/opensharing/http/ApiFailure.java index 8c1b80067..fc1b724ef 100644 --- a/server/opensharing-server-core/src/main/java/io/opensharing/http/ApiFailure.java +++ b/server/opensharing-server-core/src/main/java/io/opensharing/http/ApiFailure.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.exc.InvalidFormatException; +import io.delta.kernel.exceptions.KernelException; import io.opensharing.exception.AssetAccessDeniedException; import io.opensharing.exception.AssetNotFoundException; import io.opensharing.exception.CatalogAuthenticationException; @@ -44,6 +45,9 @@ public static ApiFailure of(Exception e) { if (e instanceof CatalogException failed) { return catalogFailure(failed); } + if (e instanceof KernelException rejected) { + return new ApiFailure(400, ErrorCodes.BAD_REQUEST, rejected.getMessage()); + } if (e instanceof JsonProcessingException unreadable) { return describe(unreadable); } diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/http/ErrorCodes.java b/server/opensharing-server-core/src/main/java/io/opensharing/http/ErrorCodes.java index 09614b1cd..55342f402 100644 --- a/server/opensharing-server-core/src/main/java/io/opensharing/http/ErrorCodes.java +++ b/server/opensharing-server-core/src/main/java/io/opensharing/http/ErrorCodes.java @@ -3,6 +3,7 @@ /** Error codes used in {@link ErrorResponse#errorCode()}. */ public final class ErrorCodes { + public static final String BAD_REQUEST = "BAD_REQUEST"; public static final String MALFORMED_REQUEST = "MALFORMED_REQUEST"; public static final String INVALID_PARAMETER_VALUE = "INVALID_PARAMETER_VALUE"; public static final String UNAUTHENTICATED = "UNAUTHENTICATED"; diff --git a/server/opensharing-server-core/src/test/java/io/opensharing/http/ApiFailureTest.java b/server/opensharing-server-core/src/test/java/io/opensharing/http/ApiFailureTest.java index f565613a2..a1b9195ff 100644 --- a/server/opensharing-server-core/src/test/java/io/opensharing/http/ApiFailureTest.java +++ b/server/opensharing-server-core/src/test/java/io/opensharing/http/ApiFailureTest.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.exc.InvalidFormatException; +import io.delta.kernel.exceptions.KernelException; import io.opensharing.auth.UserContext; import io.opensharing.catalog.Asset; import io.opensharing.catalog.AssetType; @@ -34,6 +35,17 @@ void keepsAnExplicitApiException() { 501, ErrorCodes.NOT_IMPLEMENTED, "not yet", ApiException.notImplemented("not yet")); } + @Test + void relaysKernelsReportOnATableAsABadRequest() { + assertFailure( + 400, + ErrorCodes.BAD_REQUEST, + "Cannot load table version 6 as it does not exist. The latest available version is 3.", + new KernelException( + "Cannot load table version 6 as it does not exist. The latest available version is" + + " 3.")); + } + @Test void mapsMissingAssetsAsNotFound() { assertFailure( diff --git a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolApiSupport.java b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolApiSupport.java index 8fd259458..bf4e60f54 100644 --- a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolApiSupport.java +++ b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolApiSupport.java @@ -5,6 +5,7 @@ import io.opensharing.TestServer; import io.opensharing.TestServer.Response; import io.opensharing.asset.SharedDataObjectEntity; +import io.opensharing.http.ErrorCodes; import io.opensharing.share.ShareEntity; import java.net.URI; import java.net.URLEncoder; @@ -81,6 +82,12 @@ protected static String createAndActivateRecipient(String name) { return profile.body().path("bearerToken").asText(); } + /** Kernel's report on a version or timestamp the log cannot answer. */ + protected static void assertKernelRejected(Response response) { + assertEquals(400, response.status()); + assertEquals(ErrorCodes.BAD_REQUEST, response.errorCode()); + } + /** Grants {@code recipient} SELECT on Alice's {@code share} through the provider API. */ protected static void grant(String share, String recipient) { String body = "{\"changes\":[{\"recipientName\":\"%s\",\"add\":[\"SELECT\"]}]}"; diff --git a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableMetadataIntegrationTest.java b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableMetadataIntegrationTest.java index 4d7714829..d66a5f486 100644 --- a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableMetadataIntegrationTest.java +++ b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableMetadataIntegrationTest.java @@ -58,8 +58,8 @@ void queriesLatestAndHistoricalMetadataForTable1() { // Historical version 0, and as of the first commit time, return that snapshot. assertVersion("0", metadata(endpoint + "?version=0", bearer, null)); assertVersion("0", metadata(endpoint + "?timestamp=2021-05-04T17:24:06Z", bearer, null)); - // As of before version 0 is INVALID_PARAMETER_VALUE. - assertInvalid(metadata(endpoint + "?timestamp=1970-01-01T00:00:00Z", bearer, null)); + // As of before version 0 is Kernel's report. + assertKernelRejected(metadata(endpoint + "?timestamp=1970-01-01T00:00:00Z", bearer, null)); // version and timestamp together are mutually exclusive. assertInvalid(metadata(endpoint + "?version=1×tamp=2021-05-04T17:24:06Z", bearer, null)); } diff --git a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableMetadataTest.java b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableMetadataTest.java index b8861339a..9bbcb1f71 100644 --- a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableMetadataTest.java +++ b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableMetadataTest.java @@ -80,7 +80,13 @@ void queriesGrantedTableMetadata() { assertEquals(200, atTimestamp.status()); assertEquals("1", atTimestamp.headers().get("Delta-Table-Version")); - assertInvalid(metadata(orders + "?version=9", bearer, null)); + // A version after the latest is Kernel's report. + Response afterLatest = metadata(orders + "?version=9", bearer, null); + assertEquals(400, afterLatest.status()); + assertEquals(ErrorCodes.BAD_REQUEST, afterLatest.errorCode()); + assertTrue(afterLatest + .text() + .contains("/orders: Cannot load table version 9 as it does not exist. The latest available"), afterLatest.text()); assertInvalid(metadata(orders + "?version=1×tamp=2024-01-01T01:30:00Z", bearer, null)); assertInvalid(metadata(orders + "?timestamp=not-a-timestamp", bearer, null)); diff --git a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableQueryIntegrationTest.java b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableQueryIntegrationTest.java index e73d56014..14bef0e4e 100644 --- a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableQueryIntegrationTest.java +++ b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableQueryIntegrationTest.java @@ -61,8 +61,9 @@ void queriesLatestAndHistoricalFilesForTable1() { assertVersion("0", query(endpoint, bearer, "{\"version\":0}", null)); assertVersion( "0", query(endpoint, bearer, "{\"timestamp\":\"2021-05-04T17:24:06Z\"}", null)); - // As of before version 0 is INVALID_PARAMETER_VALUE. - assertInvalid(query(endpoint, bearer, "{\"timestamp\":\"1970-01-01T00:00:00Z\"}", null)); + // As of before version 0 is Kernel's report. + assertKernelRejected( + query(endpoint, bearer, "{\"timestamp\":\"1970-01-01T00:00:00Z\"}", null)); // version and timestamp together are mutually exclusive. assertInvalid( query(endpoint, bearer, "{\"version\":1,\"timestamp\":\"2021-05-04T17:24:06Z\"}", null)); @@ -127,8 +128,8 @@ private static void queryCloudTableFiles( assertVersion("0", historical); assertEquals( 1, historical.text().split("\"version\":0", -1).length - 1); - // A version after the latest is INVALID_PARAMETER_VALUE. - assertInvalid(query(endpoint, bearer, "{\"version\":1}", null)); + // A version after the latest is Kernel's report. + assertKernelRejected(query(endpoint, bearer, "{\"version\":1}", null)); } private static void assertParquetQuery(String ndjson) { diff --git a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableQueryTest.java b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableQueryTest.java index 973fec9c6..9b0340d16 100644 --- a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableQueryTest.java +++ b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableQueryTest.java @@ -78,7 +78,11 @@ void queriesGrantedTable() { assertEquals(200, atTimestamp.status()); assertEquals("0", atTimestamp.headers().get("Delta-Table-Version")); - assertInvalid(query(events, bearer, "{\"version\":9}", null, null)); + // A version after the latest is Kernel's report. + Response afterLatest = query(events, bearer, "{\"version\":9}", null, null); + assertEquals(400, afterLatest.status()); + assertEquals(ErrorCodes.BAD_REQUEST, afterLatest.errorCode()); + assertTrue(afterLatest.text().contains("Cannot load table version 9"), afterLatest.text()); String both = "{\"version\":1,\"timestamp\":\"2024-01-01T00:30:00Z\"}"; assertInvalid(query(events, bearer, both, null, null)); assertInvalid(query(events, bearer, "{\"timestamp\":\"not-a-timestamp\"}", null, null)); diff --git a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableVersionIntegrationTest.java b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableVersionIntegrationTest.java index d7e3824cd..11b14b84b 100644 --- a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableVersionIntegrationTest.java +++ b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableVersionIntegrationTest.java @@ -38,7 +38,7 @@ void queriesLatestAndTimestampVersionsForTable1() { assertVersion("2", latest); assertTrue(latest.body().isMissingNode()); assertVersion("0", version(endpoint, bearer, "1970-01-01T00:00:00Z")); - assertInvalid(version(endpoint, bearer, "2099-01-01T00:00:00Z")); + assertKernelRejected(version(endpoint, bearer, "2099-01-01T00:00:00Z")); assertInvalid(version(endpoint, bearer, "not-a-timestamp")); } @@ -56,7 +56,7 @@ void queriesAtOrAfterTimestampsOnTheStartingTimestampTable() { assertVersion("3", version(endpoint, bearer, "2023-05-09T08:32:55.517Z")); assertVersion("4", version(endpoint, bearer, "2023-05-09T08:45:00Z")); assertVersion("5", version(endpoint, bearer, "2023-05-09T09:09:58.088Z")); - assertInvalid(version(endpoint, bearer, "2023-05-09T10:00:00Z")); + assertKernelRejected(version(endpoint, bearer, "2023-05-09T10:00:00Z")); } @Test @@ -90,7 +90,7 @@ private static void queryCloudTableVersions(String share, String catalogName, St String header = latest.headers().get("Delta-Table-Version"); assertTrue(header != null && Long.parseLong(header) >= 0, "latest version was " + header); assertVersion("0", version(endpoint, bearer, "1970-01-01T00:00:00Z")); - assertInvalid(version(endpoint, bearer, "2099-01-01T00:00:00Z")); + assertKernelRejected(version(endpoint, bearer, "2099-01-01T00:00:00Z")); } private static Response version(String path, String bearer, String startingTimestamp) { diff --git a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableVersionTest.java b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableVersionTest.java index 04cf37309..d39f6a982 100644 --- a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableVersionTest.java +++ b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolTableVersionTest.java @@ -39,7 +39,8 @@ void queriesAGrantedTableVersion() { Response afterLatest = version(salaries, bearer, "2024-01-01T03:00:01Z"); assertEquals(400, afterLatest.status()); - assertEquals(ErrorCodes.INVALID_PARAMETER_VALUE, afterLatest.errorCode()); + assertEquals(ErrorCodes.BAD_REQUEST, afterLatest.errorCode()); + assertTrue(afterLatest.text().contains("is after the latest available version")); Response invalid = version(orders, bearer, "not-a-timestamp"); assertEquals(400, invalid.status()); assertEquals(ErrorCodes.INVALID_PARAMETER_VALUE, invalid.errorCode()); diff --git a/server/opensharing-spark-e2e/src/test/java/io/opensharing/protocol/SparkSharedTableQueryIntegrationTest.java b/server/opensharing-spark-e2e/src/test/java/io/opensharing/protocol/SparkSharedTableQueryIntegrationTest.java index f64b3d178..c597cddde 100644 --- a/server/opensharing-spark-e2e/src/test/java/io/opensharing/protocol/SparkSharedTableQueryIntegrationTest.java +++ b/server/opensharing-spark-e2e/src/test/java/io/opensharing/protocol/SparkSharedTableQueryIntegrationTest.java @@ -231,7 +231,7 @@ void queriesAwsChangeDataFeed() { .load(table) .collectAsList()); assertTrue( - afterLatest.contains("startingTimestamp is after the latest table version timestamp"), + afterLatest.contains("is after the latest available version"), afterLatest); } }