diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/ProtocolTables.java b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/ProtocolTables.java index 99a511b42..d17615887 100644 --- a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/ProtocolTables.java +++ b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/ProtocolTables.java @@ -20,6 +20,7 @@ import io.opensharing.catalog.TableProperties; import io.opensharing.exception.AssetAccessDeniedException; import io.opensharing.exception.CatalogException; +import io.opensharing.exception.ChangeDataNotRecordedException; import io.opensharing.http.ApiException; import io.opensharing.http.ListResponse; import io.opensharing.http.Pagination; @@ -123,7 +124,7 @@ public long version( String tableAlias, String startingTimestamp) { ShareEntity share = protocolShares.requireGranted(recipient, shareName); - Instant timestamp = parseTimestamp("startingTimestamp", startingTimestamp); + Instant timestamp = parseTimestamp(startingTimestamp); return kernel.getVersion( findTable(share, schemaAlias, tableAlias).catalogTable(), timestamp, owner(share)); } @@ -142,7 +143,7 @@ public DeltaTableMetadataReader.Result metadata( String timestamp, String capabilities) { ShareEntity share = protocolShares.requireGranted(recipient, shareName); - Instant asOf = parseTimestamp("timestamp", timestamp); + Instant asOf = parseTimestamp(timestamp); ResolvedAsset table = findTable(share, schemaAlias, tableAlias).catalogTable(); return metadataReader.read(table, version, asOf, owner(share), capabilities); } @@ -206,6 +207,14 @@ public void query( requireUrlAccess( findTable(share, schemaAlias, tableAlias).catalogTable(), schemaAlias, tableAlias); QueryTableRequest body = request == null ? QueryTableRequest.EMPTY : request; + int selectors = + (body.version() != null ? 1 : 0) + + (body.timestamp() != null ? 1 : 0) + + (body.startingVersion() != null ? 1 : 0); + if (selectors > 1) { + throw ApiException.invalidParameter( + "Please only provide one of: version,timestamp,startingVersion."); + } if (body.startingVersion() != null || body.endingVersion() != null) { queryChanges(body, catalogTable, share, options, sink); return; @@ -229,7 +238,7 @@ public void query( options, sink, body.version(), - parseTimestamp("timestamp", body.timestamp()), + parseTimestamp(body.timestamp()), body.refreshToken(), includeRefreshToken, body.jsonPredicateHints(), @@ -251,20 +260,50 @@ public void changes( DeltaTableQueryReader.ResponseOptions options, DeltaTableQueryReader.ResponseSink sink) { ShareEntity share = protocolShares.requireGranted(recipient, shareName); + requireOneBoundary("starting", request.startingVersion(), request.startingTimestamp()); + requireOneBoundary("ending", request.endingVersion(), request.endingTimestamp()); ResolvedAsset catalogTable = requireUrlAccess( findTable(share, schemaAlias, tableAlias).catalogTable(), schemaAlias, tableAlias); - queryReader.readChangeDataFeed( - catalogTable, - owner(share), - options, - sink, - request.startingVersion(), - parseTimestamp("startingTimestamp", request.startingTimestamp()), - request.endingVersion(), - parseTimestamp("endingTimestamp", request.endingTimestamp()), - Boolean.TRUE.equals(request.includeHistoricalMetadata()), - Boolean.TRUE.equals(request.includeHistoricalProtocol())); + try { + queryReader.readChangeDataFeed( + catalogTable, + owner(share), + options, + sink, + request.startingVersion(), + parseTimestamp(request.startingTimestamp()), + request.endingVersion(), + parseTimestamp(request.endingTimestamp()), + Boolean.TRUE.equals(request.includeHistoricalMetadata()), + Boolean.TRUE.equals(request.includeHistoricalProtocol())); + } catch (ChangeDataNotRecordedException unrecorded) { + // The recipient knows the table only by the name the share gives it. + throw ApiException.permissionDenied( + "Unable to access change data feed for " + + share.getName() + + "." + + schemaAlias + + "." + + tableAlias + + ". CDF is not enabled on the original delta table for version " + + unrecorded.version() + + ". Please contact your data provider."); + } + } + + /** A change range boundary is a version or a timestamp, not both. */ + private static void requireOneBoundary(String position, Long version, String timestamp) { + if (version != null && timestamp != null) { + throw ApiException.invalidParameter( + "Multiple " + + position + + " arguments provided for CDF read. Please provide one of either " + + position + + "Timestamp or " + + position + + "Version."); + } } /** Returns {@code catalogTable} once it is known to support the {@code url} access mode. */ @@ -293,20 +332,20 @@ private void queryChanges( if (startingVersion == null) { throw ApiException.invalidParameter("endingVersion requires startingVersion"); } - if (body.version() != null || body.timestamp() != null) { - throw ApiException.invalidParameter( - "Only one of version, timestamp, and startingVersion can be set"); - } if (Boolean.TRUE.equals(body.includeRefreshToken()) || body.refreshToken() != null) { throw ApiException.invalidParameter( "includeRefreshToken and refreshToken cannot be used with startingVersion"); } if (startingVersion < 0) { - throw ApiException.invalidParameter("startingVersion must be non-negative"); + throw ApiException.invalidParameter("startingVersion cannot be negative"); } if (endingVersion != null && endingVersion < startingVersion) { throw ApiException.invalidParameter( - "endingVersion must be greater than or equal to startingVersion"); + "startingVersion(" + + startingVersion + + ") must be smaller than or equal to endingVersion(" + + endingVersion + + ")."); } queryReader.readChanges( table, @@ -379,14 +418,14 @@ private static String credentialLocation(ResolvedAsset table, String requested) } /** The ISO-8601 UTC timestamp in the parameter {@code name}, or null when absent. */ - private static Instant parseTimestamp(String name, String value) { + private static Instant parseTimestamp(String value) { if (value == null || value.isBlank()) { return null; } try { return Instant.parse(value); } catch (DateTimeParseException invalid) { - throw ApiException.invalidParameter(name + " must be an ISO-8601 UTC timestamp"); + throw ApiException.invalidParameter("Invalid timestamp: " + invalid.getMessage() + "."); } } diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/TableActions.java b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/TableActions.java index a66be2fe6..a78fdcab7 100644 --- a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/TableActions.java +++ b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/TableActions.java @@ -199,12 +199,17 @@ private static String deltaFile( public static String endStreamAction(String refreshToken, Long minUrlExpirationTimestamp) { return line( new EndStreamLine( - new EndStreamAction(refreshToken, null, minUrlExpirationTimestamp, null))); + new EndStreamAction(refreshToken, null, minUrlExpirationTimestamp, null, null))); } - /** Ends a response that failed after it began; clients treat the response as failed. */ - public static String failedEndStreamAction(String errorMessage) { - return line(new EndStreamLine(new EndStreamAction(null, null, null, errorMessage))); + /** + * Ends a response that failed after it began; clients treat the response as failed, with {@code + * httpStatusErrorCode} as the status the failure would have had. + */ + public static String failedEndStreamAction(String errorMessage, int httpStatusErrorCode) { + return line( + new EndStreamLine( + new EndStreamAction(null, null, null, errorMessage, httpStatusErrorCode))); } /** @@ -498,7 +503,8 @@ public record EndStreamLine(EndStreamAction endStreamAction) {} * *
{@code
* {"refreshToken": "", "nextPageToken": "",
- * "minUrlExpirationTimestamp": 1652144400000, "errorMessage": ""}
+ * "minUrlExpirationTimestamp": 1652144400000, "errorMessage": "",
+ * "httpStatusErrorCode": 403}
* }
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
@@ -506,5 +512,6 @@ public record EndStreamAction(
String refreshToken,
String nextPageToken,
Long minUrlExpirationTimestamp,
- String errorMessage) {}
+ String errorMessage,
+ Integer httpStatusErrorCode) {}
}
diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaTableQueryReader.java b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaTableQueryReader.java
index b5c513699..b99a7ae0f 100644
--- a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaTableQueryReader.java
+++ b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaTableQueryReader.java
@@ -45,6 +45,7 @@
import io.opensharing.asset.table.signer.UrlSigners;
import io.opensharing.auth.AuthContext;
import io.opensharing.catalog.ResolvedAsset;
+import io.opensharing.exception.ChangeDataNotRecordedException;
import io.opensharing.exception.CatalogException;
import io.opensharing.http.ApiException;
import java.io.IOException;
@@ -226,10 +227,9 @@ public void readChanges(
/**
* Reads the Change Data Feed from the start through the end version or timestamp, or the latest
- * version. Versions take precedence over timestamps, and an end after the latest version is
- * capped, as in the OSS server. Protocol and metaData lines come from the start version with
- * {@code includeHistoricalMetadata}, which also returns later metaData changes, and from the
- * latest version otherwise.
+ * version. An end after the latest version is capped. Protocol and metaData lines come from the
+ * start version with {@code includeHistoricalMetadata}, which also returns later metaData
+ * changes, and from the latest version otherwise.
*/
public void readChangeDataFeed(
ResolvedAsset table,
@@ -245,6 +245,9 @@ public void readChangeDataFeed(
try (DeltaKernel.Session session = kernel.open(table, auth, URL_TTL)) {
SnapshotImpl latest = (SnapshotImpl) DeltaSnapshots.open(session, null, null);
TableImpl delta = (TableImpl) session.table();
+ if (endingTimestamp != null) {
+ requireEndingTimestampAtMostLatest(endingTimestamp, latest, session);
+ }
long start;
if (startingVersion != null) {
start = startingVersion;
@@ -270,7 +273,7 @@ public void readChangeDataFeed(
long resolvedEnd = end == null ? latest.getVersion() : Math.min(end, latest.getVersion());
SnapshotImpl first = (SnapshotImpl) DeltaSnapshots.open(session, start, null);
if (!changeDataFeedEnabled(first.getMetadata())) {
- throw changeDataNotRecorded(start, start, resolvedEnd);
+ throw new ChangeDataNotRecordedException(start);
}
changes(
session,
@@ -283,6 +286,25 @@ public void readChangeDataFeed(
}
}
+ /**
+ * An ending timestamp after the latest commit is the recipient's mistake, and the table's latest
+ * commit time says what to ask for instead.
+ */
+ private static void requireEndingTimestampAtMostLatest(
+ Instant endingTimestamp, SnapshotImpl latest, DeltaKernel.Session session) {
+ long latestTimestamp = latest.getTimestamp(session.engine());
+ if (endingTimestamp.toEpochMilli() > latestTimestamp) {
+ String latestInstant = Instant.ofEpochMilli(latestTimestamp).toString();
+ throw ApiException.invalidParameter(
+ "The provided timestamp("
+ + endingTimestamp
+ + ") is after the latest table version available to this table ("
+ + latestInstant
+ + "). Please use a timestamp less than or equal to "
+ + latestInstant);
+ }
+ }
+
private static long versionBeforeOrAt(
TableImpl table, DeltaKernel.Session session, Instant endingTimestamp) {
try {
@@ -396,7 +418,7 @@ private static void write(ResponseWriter out, ChangeRange range, Commit commit)
}
if (commit.metadata != null) {
if (!changeDataFeedEnabled(commit.metadata)) {
- throw changeDataNotRecorded(version, range.start(), range.end());
+ throw new ChangeDataNotRecordedException(version);
}
if (range.includeHistoricalMetadata() && later) {
out.metadata(commit.metadata, version, null);
@@ -417,17 +439,6 @@ private static boolean changeDataFeedEnabled(Metadata metadata) {
return TableConfig.CHANGE_DATA_FEED_ENABLED.fromMetadata(metadata);
}
- private static ApiException changeDataNotRecorded(long version, long start, long end) {
- return ApiException.invalidParameter(
- "Error getting change data for range ["
- + start
- + ", "
- + end
- + "] as change data was not recorded for version ["
- + version
- + "]");
- }
-
// MERGE can rewrite files with dataChange but change no rows; its metrics say so, and no cdc
// is written. Missing metrics do not skip.
private static boolean noOpMerge(Row commitInfo) {
diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/exception/ChangeDataNotRecordedException.java b/server/opensharing-server-core/src/main/java/io/opensharing/exception/ChangeDataNotRecordedException.java
new file mode 100644
index 000000000..256d46da3
--- /dev/null
+++ b/server/opensharing-server-core/src/main/java/io/opensharing/exception/ChangeDataNotRecordedException.java
@@ -0,0 +1,20 @@
+package io.opensharing.exception;
+
+/**
+ * A Change Data Feed read reached a table version that did not record change data. The protocol
+ * layer, which knows the table's shared name, answers it as PERMISSION_DENIED.
+ */
+public class ChangeDataNotRecordedException extends RuntimeException {
+
+ private final long version;
+
+ public ChangeDataNotRecordedException(long version) {
+ super("change data was not recorded for version " + version);
+ this.version = version;
+ }
+
+ /** The first version in the range that did not record change data. */
+ public long version() {
+ return version;
+ }
+}
diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/protocol/NdjsonResponse.java b/server/opensharing-server-core/src/main/java/io/opensharing/protocol/NdjsonResponse.java
index e4216c598..9b99422d8 100644
--- a/server/opensharing-server-core/src/main/java/io/opensharing/protocol/NdjsonResponse.java
+++ b/server/opensharing-server-core/src/main/java/io/opensharing/protocol/NdjsonResponse.java
@@ -25,12 +25,13 @@
* table version and format, and each line as soon as it is read, without waiting for the client to
* take the earlier ones, so lines a slow client has not received yet queue in memory. A response
* that fails before then is an ordinary error response. One that fails later has already sent its
- * status, so it ends with an endStreamAction carrying the error when the client asked for one, and
- * is cut short otherwise.
+ * status, so it ends with an endStreamAction carrying the error and its status when the client
+ * asked for one, and with a line that is not JSON otherwise.
*/
final class NdjsonResponse implements ResponseSink {
private static final Logger log = LoggerFactory.getLogger(NdjsonResponse.class);
+ private static final int MAX_ERROR_CHARS = 500;
private final ResponseOptions options;
private final CompletableFuture