Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,9 @@ public long getVersion(ResolvedAsset table, Instant startingTimestamp, AuthConte

static long versionAtOrAfter(TableImpl table, Engine engine, Instant startingTimestamp) {
Snapshot latest = latestSnapshot(table, engine);
if (startingTimestamp == null) {
return latest.getVersion();
}
if (startingTimestamp.toEpochMilli() > latest.getTimestamp(engine)) {
throw ApiException.invalidParameter(
"startingTimestamp is after the latest table version timestamp");
}
return table.getVersionAtOrAfterTimestamp(engine, startingTimestamp.toEpochMilli());
return startingTimestamp == null
? latest.getVersion()
: table.getVersionAtOrAfterTimestamp(engine, startingTimestamp.toEpochMilli());
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import io.delta.kernel.internal.SnapshotImpl;
import io.delta.kernel.internal.checksum.CRCInfo;
import io.opensharing.http.ApiException;
import io.opensharing.http.ErrorCodes;
import java.time.Instant;

/** Opens a Kernel snapshot for a version, timestamp, or the latest log, and reads its checksum. */
Expand All @@ -14,30 +15,31 @@ private DeltaSnapshots() {}

static Snapshot open(DeltaKernel.Session session, Long version, Instant timestamp) {
Snapshot latest = DeltaKernel.latestSnapshot(session.table(), session.engine());
if (version != null) {
if (version > latest.getVersion()) {
throw ApiException.invalidParameter(
"version " + version + " is after the latest table version " + latest.getVersion());
}
try {
return session.table().getSnapshotAsOfVersion(session.engine(), version);
} catch (KernelException | IllegalArgumentException unavailable) {
throw unavailable(unavailable, "version " + version + " is no longer available");
}
if (version == null && timestamp == null) {
return latest;
}
if (timestamp != null) {
if (timestamp.toEpochMilli() > latest.getTimestamp(session.engine())) {
throw ApiException.invalidParameter(
"timestamp is after the latest table version timestamp");
}
try {
return session.table().getSnapshotAsOfTimestamp(session.engine(), timestamp.toEpochMilli());
} catch (KernelException | IllegalArgumentException unavailable) {
throw unavailable(
unavailable, "timestamp is before the earliest available table version timestamp");
}
if (version != null && version > latest.getVersion()) {
// Kernel's message, table path included, without the search back through the log that it
// makes for a version this far ahead, which takes minutes on object storage.
throw new ApiException(
400,
ErrorCodes.BAD_REQUEST,
session.table().getPath(session.engine())
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
+ ": Cannot load table version "
+ version
+ " as it does not exist. The latest available version is "
+ latest.getVersion()
+ ".");
}
try {
return version != null
? session.table().getSnapshotAsOfVersion(session.engine(), version)
: session.table().getSnapshotAsOfTimestamp(session.engine(), timestamp.toEpochMilli());
} catch (KernelException unavailable) {
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
throw DeltaKernel.unsupportedProtocol(unavailable)
? DeltaKernel.unreadableTable(unavailable)
: unavailable;
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
}
return latest;
}

/**
Expand All @@ -50,12 +52,4 @@ static CRCInfo crcFor(SnapshotImpl snapshot) {
.filter(crc -> crc.getVersion() == snapshot.getVersion())
.orElse(null);
}

/** An earlier snapshot can need a protocol Kernel cannot read even when the latest does not. */
private static ApiException unavailable(RuntimeException failed, String message) {
if (failed instanceof KernelException kernel && DeltaKernel.unsupportedProtocol(kernel)) {
return DeltaKernel.unreadableTable(kernel);
}
return ApiException.invalidParameter(message);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.exc.InvalidFormatException;
import io.delta.kernel.exceptions.KernelException;
import io.opensharing.exception.AssetAccessDeniedException;
import io.opensharing.exception.AssetNotFoundException;
import io.opensharing.exception.CatalogAuthenticationException;
Expand Down Expand Up @@ -44,6 +45,9 @@ public static ApiFailure of(Exception e) {
if (e instanceof CatalogException failed) {
return catalogFailure(failed);
}
if (e instanceof KernelException rejected) {
return new ApiFailure(400, ErrorCodes.BAD_REQUEST, rejected.getMessage());
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
}
if (e instanceof JsonProcessingException unreadable) {
return describe(unreadable);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
/** Error codes used in {@link ErrorResponse#errorCode()}. */
public final class ErrorCodes {

public static final String BAD_REQUEST = "BAD_REQUEST";
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
public static final String MALFORMED_REQUEST = "MALFORMED_REQUEST";
public static final String INVALID_PARAMETER_VALUE = "INVALID_PARAMETER_VALUE";
public static final String UNAUTHENTICATED = "UNAUTHENTICATED";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.exc.InvalidFormatException;
import io.delta.kernel.exceptions.KernelException;
import io.opensharing.auth.UserContext;
import io.opensharing.catalog.Asset;
import io.opensharing.catalog.AssetType;
Expand Down Expand Up @@ -34,6 +35,17 @@ void keepsAnExplicitApiException() {
501, ErrorCodes.NOT_IMPLEMENTED, "not yet", ApiException.notImplemented("not yet"));
}

@Test
void relaysKernelsReportOnATableAsABadRequest() {
assertFailure(
400,
ErrorCodes.BAD_REQUEST,
"Cannot load table version 6 as it does not exist. The latest available version is 3.",
new KernelException(
"Cannot load table version 6 as it does not exist. The latest available version is"
+ " 3."));
}

@Test
void mapsMissingAssetsAsNotFound() {
assertFailure(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import io.opensharing.TestServer;
import io.opensharing.TestServer.Response;
import io.opensharing.asset.SharedDataObjectEntity;
import io.opensharing.http.ErrorCodes;
import io.opensharing.share.ShareEntity;
import java.net.URI;
import java.net.URLEncoder;
Expand Down Expand Up @@ -81,6 +82,12 @@ protected static String createAndActivateRecipient(String name) {
return profile.body().path("bearerToken").asText();
}

/** Kernel's report on a version or timestamp the log cannot answer. */
protected static void assertKernelRejected(Response response) {
assertEquals(400, response.status());
assertEquals(ErrorCodes.BAD_REQUEST, response.errorCode());
}

/** Grants {@code recipient} SELECT on Alice's {@code share} through the provider API. */
protected static void grant(String share, String recipient) {
String body = "{\"changes\":[{\"recipientName\":\"%s\",\"add\":[\"SELECT\"]}]}";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ void queriesLatestAndHistoricalMetadataForTable1() {
// Historical version 0, and as of the first commit time, return that snapshot.
assertVersion("0", metadata(endpoint + "?version=0", bearer, null));
assertVersion("0", metadata(endpoint + "?timestamp=2021-05-04T17:24:06Z", bearer, null));
// As of before version 0 is INVALID_PARAMETER_VALUE.
assertInvalid(metadata(endpoint + "?timestamp=1970-01-01T00:00:00Z", bearer, null));
// As of before version 0 is Kernel's report.
assertKernelRejected(metadata(endpoint + "?timestamp=1970-01-01T00:00:00Z", bearer, null));
// version and timestamp together are mutually exclusive.
assertInvalid(metadata(endpoint + "?version=1&timestamp=2021-05-04T17:24:06Z", bearer, null));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,13 @@ void queriesGrantedTableMetadata() {
assertEquals(200, atTimestamp.status());
assertEquals("1", atTimestamp.headers().get("Delta-Table-Version"));

assertInvalid(metadata(orders + "?version=9", bearer, null));
// A version after the latest is Kernel's report.
Response afterLatest = metadata(orders + "?version=9", bearer, null);
assertEquals(400, afterLatest.status());
assertEquals(ErrorCodes.BAD_REQUEST, afterLatest.errorCode());
assertTrue(afterLatest
.text()
.contains("/orders: Cannot load table version 9 as it does not exist. The latest available"), afterLatest.text());
assertInvalid(metadata(orders + "?version=1&timestamp=2024-01-01T01:30:00Z", bearer, null));
assertInvalid(metadata(orders + "?timestamp=not-a-timestamp", bearer, null));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,9 @@ void queriesLatestAndHistoricalFilesForTable1() {
assertVersion("0", query(endpoint, bearer, "{\"version\":0}", null));
assertVersion(
"0", query(endpoint, bearer, "{\"timestamp\":\"2021-05-04T17:24:06Z\"}", null));
// As of before version 0 is INVALID_PARAMETER_VALUE.
assertInvalid(query(endpoint, bearer, "{\"timestamp\":\"1970-01-01T00:00:00Z\"}", null));
// As of before version 0 is Kernel's report.
assertKernelRejected(
query(endpoint, bearer, "{\"timestamp\":\"1970-01-01T00:00:00Z\"}", null));
// version and timestamp together are mutually exclusive.
assertInvalid(
query(endpoint, bearer, "{\"version\":1,\"timestamp\":\"2021-05-04T17:24:06Z\"}", null));
Expand Down Expand Up @@ -127,8 +128,8 @@ private static void queryCloudTableFiles(
assertVersion("0", historical);
assertEquals(
1, historical.text().split("\"version\":0", -1).length - 1);
// A version after the latest is INVALID_PARAMETER_VALUE.
assertInvalid(query(endpoint, bearer, "{\"version\":1}", null));
// A version after the latest is Kernel's report.
assertKernelRejected(query(endpoint, bearer, "{\"version\":1}", null));
}

private static void assertParquetQuery(String ndjson) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,11 @@ void queriesGrantedTable() {
assertEquals(200, atTimestamp.status());
assertEquals("0", atTimestamp.headers().get("Delta-Table-Version"));

assertInvalid(query(events, bearer, "{\"version\":9}", null, null));
// A version after the latest is Kernel's report.
Response afterLatest = query(events, bearer, "{\"version\":9}", null, null);
assertEquals(400, afterLatest.status());
assertEquals(ErrorCodes.BAD_REQUEST, afterLatest.errorCode());
assertTrue(afterLatest.text().contains("Cannot load table version 9"), afterLatest.text());
String both = "{\"version\":1,\"timestamp\":\"2024-01-01T00:30:00Z\"}";
assertInvalid(query(events, bearer, both, null, null));
assertInvalid(query(events, bearer, "{\"timestamp\":\"not-a-timestamp\"}", null, null));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ void queriesLatestAndTimestampVersionsForTable1() {
assertVersion("2", latest);
assertTrue(latest.body().isMissingNode());
assertVersion("0", version(endpoint, bearer, "1970-01-01T00:00:00Z"));
assertInvalid(version(endpoint, bearer, "2099-01-01T00:00:00Z"));
assertKernelRejected(version(endpoint, bearer, "2099-01-01T00:00:00Z"));
assertInvalid(version(endpoint, bearer, "not-a-timestamp"));
}

Expand All @@ -56,7 +56,7 @@ void queriesAtOrAfterTimestampsOnTheStartingTimestampTable() {
assertVersion("3", version(endpoint, bearer, "2023-05-09T08:32:55.517Z"));
assertVersion("4", version(endpoint, bearer, "2023-05-09T08:45:00Z"));
assertVersion("5", version(endpoint, bearer, "2023-05-09T09:09:58.088Z"));
assertInvalid(version(endpoint, bearer, "2023-05-09T10:00:00Z"));
assertKernelRejected(version(endpoint, bearer, "2023-05-09T10:00:00Z"));
}

@Test
Expand Down Expand Up @@ -90,7 +90,7 @@ private static void queryCloudTableVersions(String share, String catalogName, St
String header = latest.headers().get("Delta-Table-Version");
assertTrue(header != null && Long.parseLong(header) >= 0, "latest version was " + header);
assertVersion("0", version(endpoint, bearer, "1970-01-01T00:00:00Z"));
assertInvalid(version(endpoint, bearer, "2099-01-01T00:00:00Z"));
assertKernelRejected(version(endpoint, bearer, "2099-01-01T00:00:00Z"));
}

private static Response version(String path, String bearer, String startingTimestamp) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ void queriesAGrantedTableVersion() {

Response afterLatest = version(salaries, bearer, "2024-01-01T03:00:01Z");
assertEquals(400, afterLatest.status());
assertEquals(ErrorCodes.INVALID_PARAMETER_VALUE, afterLatest.errorCode());
assertEquals(ErrorCodes.BAD_REQUEST, afterLatest.errorCode());
assertTrue(afterLatest.text().contains("is after the latest available version"));
Response invalid = version(orders, bearer, "not-a-timestamp");
assertEquals(400, invalid.status());
assertEquals(ErrorCodes.INVALID_PARAMETER_VALUE, invalid.errorCode());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ void queriesAwsChangeDataFeed() {
.load(table)
.collectAsList());
assertTrue(
afterLatest.contains("startingTimestamp is after the latest table version timestamp"),
afterLatest.contains("is after the latest available version"),
afterLatest);
}
}
Expand Down
Loading