Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import io.opensharing.catalog.TableProperties;
import io.opensharing.exception.AssetAccessDeniedException;
import io.opensharing.exception.CatalogException;
import io.opensharing.exception.ChangeDataNotRecordedException;
import io.opensharing.http.ApiException;
import io.opensharing.http.ListResponse;
import io.opensharing.http.Pagination;
Expand Down Expand Up @@ -123,7 +124,7 @@ public long version(
String tableAlias,
String startingTimestamp) {
ShareEntity share = protocolShares.requireGranted(recipient, shareName);
Instant timestamp = parseTimestamp("startingTimestamp", startingTimestamp);
Instant timestamp = parseTimestamp(startingTimestamp);
return kernel.getVersion(
findTable(share, schemaAlias, tableAlias).catalogTable(), timestamp, owner(share));
}
Expand All @@ -142,7 +143,7 @@ public DeltaTableMetadataReader.Result metadata(
String timestamp,
String capabilities) {
ShareEntity share = protocolShares.requireGranted(recipient, shareName);
Instant asOf = parseTimestamp("timestamp", timestamp);
Instant asOf = parseTimestamp(timestamp);
ResolvedAsset table = findTable(share, schemaAlias, tableAlias).catalogTable();
return metadataReader.read(table, version, asOf, owner(share), capabilities);
}
Expand Down Expand Up @@ -206,6 +207,14 @@ public void query(
requireUrlAccess(
findTable(share, schemaAlias, tableAlias).catalogTable(), schemaAlias, tableAlias);
QueryTableRequest body = request == null ? QueryTableRequest.EMPTY : request;
int selectors =
(body.version() != null ? 1 : 0)
+ (body.timestamp() != null ? 1 : 0)
+ (body.startingVersion() != null ? 1 : 0);
if (selectors > 1) {
throw ApiException.invalidParameter(
"Please only provide one of: version,timestamp,startingVersion.");
}
if (body.startingVersion() != null || body.endingVersion() != null) {
queryChanges(body, catalogTable, share, options, sink);
return;
Expand All @@ -229,7 +238,7 @@ public void query(
options,
sink,
body.version(),
parseTimestamp("timestamp", body.timestamp()),
parseTimestamp(body.timestamp()),
body.refreshToken(),
includeRefreshToken,
body.jsonPredicateHints(),
Expand All @@ -251,20 +260,50 @@ public void changes(
DeltaTableQueryReader.ResponseOptions options,
DeltaTableQueryReader.ResponseSink sink) {
ShareEntity share = protocolShares.requireGranted(recipient, shareName);
requireOneBoundary("starting", request.startingVersion(), request.startingTimestamp());
requireOneBoundary("ending", request.endingVersion(), request.endingTimestamp());
ResolvedAsset catalogTable =
requireUrlAccess(
findTable(share, schemaAlias, tableAlias).catalogTable(), schemaAlias, tableAlias);
queryReader.readChangeDataFeed(
catalogTable,
owner(share),
options,
sink,
request.startingVersion(),
parseTimestamp("startingTimestamp", request.startingTimestamp()),
request.endingVersion(),
parseTimestamp("endingTimestamp", request.endingTimestamp()),
Boolean.TRUE.equals(request.includeHistoricalMetadata()),
Boolean.TRUE.equals(request.includeHistoricalProtocol()));
try {
queryReader.readChangeDataFeed(
catalogTable,
owner(share),
options,
sink,
request.startingVersion(),
parseTimestamp(request.startingTimestamp()),
request.endingVersion(),
parseTimestamp(request.endingTimestamp()),
Boolean.TRUE.equals(request.includeHistoricalMetadata()),
Boolean.TRUE.equals(request.includeHistoricalProtocol()));
} catch (ChangeDataNotRecordedException unrecorded) {
// The recipient knows the table only by the name the share gives it.
throw ApiException.permissionDenied(
"Unable to access change data feed for "
+ share.getName()
+ "."
+ schemaAlias
+ "."
+ tableAlias
+ ". CDF is not enabled on the original delta table for version "
+ unrecorded.version()
+ ". Please contact your data provider.");
}
}

/** A change range boundary is a version or a timestamp, not both. */
private static void requireOneBoundary(String position, Long version, String timestamp) {
if (version != null && timestamp != null) {
throw ApiException.invalidParameter(
"Multiple "
+ position
+ " arguments provided for CDF read. Please provide one of either "
+ position
+ "Timestamp or "
+ position
+ "Version.");
}
}

/** Returns {@code catalogTable} once it is known to support the {@code url} access mode. */
Expand Down Expand Up @@ -293,20 +332,20 @@ private void queryChanges(
if (startingVersion == null) {
throw ApiException.invalidParameter("endingVersion requires startingVersion");
}
if (body.version() != null || body.timestamp() != null) {
throw ApiException.invalidParameter(
"Only one of version, timestamp, and startingVersion can be set");
}
if (Boolean.TRUE.equals(body.includeRefreshToken()) || body.refreshToken() != null) {
throw ApiException.invalidParameter(
"includeRefreshToken and refreshToken cannot be used with startingVersion");
}
if (startingVersion < 0) {
throw ApiException.invalidParameter("startingVersion must be non-negative");
throw ApiException.invalidParameter("startingVersion cannot be negative");
}
if (endingVersion != null && endingVersion < startingVersion) {
throw ApiException.invalidParameter(
"endingVersion must be greater than or equal to startingVersion");
"startingVersion("
+ startingVersion
+ ") must be smaller than or equal to endingVersion("
+ endingVersion
+ ").");
}
queryReader.readChanges(
table,
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
Expand Down Expand Up @@ -379,14 +418,14 @@ private static String credentialLocation(ResolvedAsset table, String requested)
}

/** The ISO-8601 UTC timestamp in the parameter {@code name}, or null when absent. */
private static Instant parseTimestamp(String name, String value) {
private static Instant parseTimestamp(String value) {
if (value == null || value.isBlank()) {
return null;
}
try {
return Instant.parse(value);
} catch (DateTimeParseException invalid) {
throw ApiException.invalidParameter(name + " must be an ISO-8601 UTC timestamp");
throw ApiException.invalidParameter("Invalid timestamp: " + invalid.getMessage() + ".");
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -199,12 +199,17 @@ private static String deltaFile(
public static String endStreamAction(String refreshToken, Long minUrlExpirationTimestamp) {
return line(
new EndStreamLine(
new EndStreamAction(refreshToken, null, minUrlExpirationTimestamp, null)));
new EndStreamAction(refreshToken, null, minUrlExpirationTimestamp, null, null)));
}

/** Ends a response that failed after it began; clients treat the response as failed. */
public static String failedEndStreamAction(String errorMessage) {
return line(new EndStreamLine(new EndStreamAction(null, null, null, errorMessage)));
/**
* Ends a response that failed after it began; clients treat the response as failed, with {@code
* httpStatusErrorCode} as the status the failure would have had.
*/
public static String failedEndStreamAction(String errorMessage, int httpStatusErrorCode) {
return line(
new EndStreamLine(
new EndStreamAction(null, null, null, errorMessage, httpStatusErrorCode)));
}

/**
Expand Down Expand Up @@ -498,13 +503,15 @@ public record EndStreamLine(EndStreamAction endStreamAction) {}
*
* <pre>{@code
* {"refreshToken": "<refresh token>", "nextPageToken": "<page token>",
* "minUrlExpirationTimestamp": 1652144400000, "errorMessage": "<error>"}
* "minUrlExpirationTimestamp": 1652144400000, "errorMessage": "<error>",
* "httpStatusErrorCode": 403}
* }</pre>
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public record EndStreamAction(
String refreshToken,
String nextPageToken,
Long minUrlExpirationTimestamp,
String errorMessage) {}
String errorMessage,
Integer httpStatusErrorCode) {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import io.opensharing.asset.table.signer.UrlSigners;
import io.opensharing.auth.AuthContext;
import io.opensharing.catalog.ResolvedAsset;
import io.opensharing.exception.ChangeDataNotRecordedException;
import io.opensharing.exception.CatalogException;
import io.opensharing.http.ApiException;
import java.io.IOException;
Expand Down Expand Up @@ -226,10 +227,9 @@ public void readChanges(

/**
* Reads the Change Data Feed from the start through the end version or timestamp, or the latest
* version. Versions take precedence over timestamps, and an end after the latest version is
* capped, as in the OSS server. Protocol and metaData lines come from the start version with
* {@code includeHistoricalMetadata}, which also returns later metaData changes, and from the
* latest version otherwise.
* version. An end after the latest version is capped. Protocol and metaData lines come from the
* start version with {@code includeHistoricalMetadata}, which also returns later metaData
* changes, and from the latest version otherwise.
*/
public void readChangeDataFeed(
ResolvedAsset table,
Expand All @@ -245,6 +245,9 @@ public void readChangeDataFeed(
try (DeltaKernel.Session session = kernel.open(table, auth, URL_TTL)) {
SnapshotImpl latest = (SnapshotImpl) DeltaSnapshots.open(session, null, null);
TableImpl delta = (TableImpl) session.table();
if (endingTimestamp != null) {
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
requireEndingTimestampAtMostLatest(endingTimestamp, latest, session);
}
long start;
if (startingVersion != null) {
start = startingVersion;
Expand All @@ -270,7 +273,7 @@ public void readChangeDataFeed(
long resolvedEnd = end == null ? latest.getVersion() : Math.min(end, latest.getVersion());
SnapshotImpl first = (SnapshotImpl) DeltaSnapshots.open(session, start, null);
if (!changeDataFeedEnabled(first.getMetadata())) {
throw changeDataNotRecorded(start, start, resolvedEnd);
throw new ChangeDataNotRecordedException(start);
}
changes(
session,
Expand All @@ -283,6 +286,25 @@ public void readChangeDataFeed(
}
}

/**
* An ending timestamp after the latest commit is the recipient's mistake, and the table's latest
* commit time says what to ask for instead.
*/
private static void requireEndingTimestampAtMostLatest(
Instant endingTimestamp, SnapshotImpl latest, DeltaKernel.Session session) {
long latestTimestamp = latest.getTimestamp(session.engine());
if (endingTimestamp.toEpochMilli() > latestTimestamp) {
String latestInstant = Instant.ofEpochMilli(latestTimestamp).toString();
throw ApiException.invalidParameter(
"The provided timestamp("
+ endingTimestamp
+ ") is after the latest table version available to this table ("
+ latestInstant
+ "). Please use a timestamp less than or equal to "
+ latestInstant);
}
}

private static long versionBeforeOrAt(
TableImpl table, DeltaKernel.Session session, Instant endingTimestamp) {
try {
Expand Down Expand Up @@ -396,7 +418,7 @@ private static void write(ResponseWriter out, ChangeRange range, Commit commit)
}
if (commit.metadata != null) {
if (!changeDataFeedEnabled(commit.metadata)) {
throw changeDataNotRecorded(version, range.start(), range.end());
throw new ChangeDataNotRecordedException(version);
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
}
if (range.includeHistoricalMetadata() && later) {
out.metadata(commit.metadata, version, null);
Expand All @@ -417,17 +439,6 @@ private static boolean changeDataFeedEnabled(Metadata metadata) {
return TableConfig.CHANGE_DATA_FEED_ENABLED.fromMetadata(metadata);
}

private static ApiException changeDataNotRecorded(long version, long start, long end) {
return ApiException.invalidParameter(
"Error getting change data for range ["
+ start
+ ", "
+ end
+ "] as change data was not recorded for version ["
+ version
+ "]");
}

// MERGE can rewrite files with dataChange but change no rows; its metrics say so, and no cdc
// is written. Missing metrics do not skip.
private static boolean noOpMerge(Row commitInfo) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package io.opensharing.exception;

/**
* A Change Data Feed read reached a table version that did not record change data. The protocol
* layer, which knows the table's shared name, answers it as PERMISSION_DENIED.
*/
public class ChangeDataNotRecordedException extends RuntimeException {

private final long version;

public ChangeDataNotRecordedException(long version) {
super("change data was not recorded for version " + version);
this.version = version;
}

/** The first version in the range that did not record change data. */
public long version() {
return version;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,13 @@
* table version and format, and each line as soon as it is read, without waiting for the client to
* take the earlier ones, so lines a slow client has not received yet queue in memory. A response
* that fails before then is an ordinary error response. One that fails later has already sent its
* status, so it ends with an endStreamAction carrying the error when the client asked for one, and
* is cut short otherwise.
* status, so it ends with an endStreamAction carrying the error and its status when the client
* asked for one, and with a line that is not JSON otherwise.
*/
final class NdjsonResponse implements ResponseSink {

private static final Logger log = LoggerFactory.getLogger(NdjsonResponse.class);
private static final int MAX_ERROR_CHARS = 500;

private final ResponseOptions options;
private final CompletableFuture<HttpResponse> response = new CompletableFuture<>();
Expand Down Expand Up @@ -89,17 +90,29 @@ private void fail(Throwable failure) {
response.completeExceptionally(failure);
} else if (!writer.isOpen()) {
return;
} else if (options.includeEndStreamAction()) {
} else {
// ApiFailure logs catalog and unhandled failures, as it does for an error response.
String message =
ApiFailure failed =
ApiFailure.ofRecipientRequest(
failure instanceof Exception e ? e : new RuntimeException(failure))
.message();
writer.tryWrite(HttpData.ofUtf8(TableActions.failedEndStreamAction(message)));
failure instanceof Exception e ? e : new RuntimeException(failure));
String described = failed.errorCode() + ": " + failed.message();
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
if (options.includeEndStreamAction()) {
writer.tryWrite(
HttpData.ofUtf8(TableActions.failedEndStreamAction(described, failed.status())));
} else {
// The status is sent, so the client is made to fail on a line that is not JSON.
log.error("Query response failed after it began streaming", failure);
writer.tryWrite(HttpData.ofUtf8(invalidJson(described) + "\n"));
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
}
writer.close();
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
} else {
log.error("Query response failed after it began streaming", failure);
writer.abort(failure);
}
}

/** A token that is not JSON and carries the error, which a client's parser reports. */
private static String invalidJson(String error) {
String described =
"Forcing invalid json to fail streaming rpc due to error "
+ error.substring(0, Math.min(error.length(), MAX_ERROR_CHARS));
return described.replaceAll("[^A-Za-z0-9_]", "_") + "{";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -179,9 +179,9 @@ void encodesEndStreamAction() {
TableActions.endStreamAction(null, null));
assertEquals(
"""
{"endStreamAction":{"errorMessage":"bad version"}}
{"endStreamAction":{"errorMessage":"bad version","httpStatusErrorCode":400}}
""",
TableActions.failedEndStreamAction("bad version"));
TableActions.failedEndStreamAction("bad version", 400));
}

@Test
Expand Down
Loading
Loading