Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@

import io.delta.kernel.internal.actions.Protocol;
import io.opensharing.http.ApiException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -50,40 +52,102 @@ public static boolean includeEndStreamAction(String header) {
return "true".equals(parse(header).get("includeendstreamaction"));
}

/**
* The Delta table properties that stand for reader features, in the order they are checked, each
* with the reader feature that covers it and the value that turns the feature off.
*/
private static final List<PropertyWithOffValue> FEATURE_PROPERTIES =
List.of(
new PropertyWithOffValue("delta.columnMapping.mode", "columnmapping", "none"),
new PropertyWithOffValue("delta.enableDeletionVectors", "deletionvectors", "false"));

private record PropertyWithOffValue(String property, String feature, String offValue) {}

/** What to do about table features the client did not advertise. */
private static final String ADVERTISE_OR_DISABLE =
"Query with responseFormat=delta and advertise the features in delta-sharing-capabilities,"
+ " or ask your data provider to disable them.";

/** Deletion vectors stay in a protocol after they are turned off; their property decides. */
private static final String DELETION_VECTORS = "deletionvectors";

/** What a client declared in {@code delta-sharing-capabilities}, parsed once per request. */
public record Declared(Set<ResponseFormat> formats, Set<String> readerFeatures) {}

public static Declared declared(String header) {
Map<String, String> capabilities = parse(header);
String value = capabilities.get("readerfeatures");
Set<String> readerFeatures = new LinkedHashSet<>();
if (value != null) {
for (String token : value.split(",")) {
readerFeatures.add(token.trim());
}
}
return new Declared(responseFormats(capabilities), readerFeatures);
}

/**
* A table whose Delta properties enable a reader feature the client did not advertise cannot be
* served to it: a property may only be present with the value that turns its feature off, in any
* case as Delta reads it. This covers the table's version {@code version}.
*/
public static void requireSupportedProperties(
Declared declared, Map<String, String> configuration, long version) {
if (configuration == null || configuration.isEmpty()) {
return;
}
List<String> unsupported = new ArrayList<>();
for (PropertyWithOffValue checked : FEATURE_PROPERTIES) {
String value = configuration.get(checked.property());
if (value != null
&& !declared.readerFeatures().contains(checked.feature())
&& !checked.offValue().equalsIgnoreCase(value.trim())) {
unsupported.add(checked.property());
}
}
if (!unsupported.isEmpty()) {
throw ApiException.unsupportedFeatures(unsupported, version, ADVERTISE_OR_DISABLE);
}
}

/**
* A parquet response only carries protocols with reader version 1. A client that accepts delta
* must advertise every reader feature the table requires; one that accepts only parquet gets
* delta for an advanced table and raises the error itself. Advertised features the server does
* not know are ignored. Metadata does not enforce this so clients can inspect protocol before
* querying files.
* querying files. {@code version} is the table version {@code protocol} is of.
*/
public static void requireReaderFeatures(
String header, ResponseFormat format, Protocol protocol) {
Declared declared, ResponseFormat format, Protocol protocol, long version) {
Set<String> required = protocol.getReaderFeatures();
if (format != ResponseFormat.DELTA) {
if (protocol.getMinReaderVersion() > 1) {
throw ApiException.invalidParameter(
"The table requires reader features that are only available in delta format");
throw ApiException.unsupportedFeatures(
required == null || required.isEmpty()
? List.of("reader version " + protocol.getMinReaderVersion())
: lowerCase(required),
version,
ADVERTISE_OR_DISABLE);
}
return;
}
if (!responseFormats(parse(header)).contains(ResponseFormat.DELTA)) {
return;
}
Set<String> required = protocol.getReaderFeatures();
if (required == null || required.isEmpty()) {
if (!declared.formats().contains(ResponseFormat.DELTA) || required == null) {
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
return;
}
Set<String> advertised = advertisedReaderFeatures(header);
for (String feature : required) {
if (!advertised.contains(feature.toLowerCase(Locale.ROOT))) {
throw ApiException.invalidParameter(
"The table requires reader feature '"
+ feature
+ "' which was not specified in delta-sharing-capabilities");
}
List<String> unadvertised =
lowerCase(required).stream()
.filter(feature -> !feature.equals(DELETION_VECTORS))
Comment thread
littlegrasscao marked this conversation as resolved.
Resolved
.filter(feature -> !declared.readerFeatures().contains(feature))
.toList();
if (!unadvertised.isEmpty()) {
throw ApiException.unsupportedFeatures(unadvertised, version, ADVERTISE_OR_DISABLE);
}
}

private static List<String> lowerCase(Set<String> features) {
return features.stream().map(feature -> feature.toLowerCase(Locale.ROOT)).toList();
}

/**
* The capabilities this server knows. Clients advertise ones it does not, such as asyncquery,
* and may send parts without a value; both are ignored.
Expand Down Expand Up @@ -133,16 +197,4 @@ static Set<ResponseFormat> responseFormats(Map<String, String> capabilities) {
}
return requested;
}

private static Set<String> advertisedReaderFeatures(String header) {
String value = parse(header).get("readerfeatures");
if (value == null) {
return Set.of();
}
Set<String> advertised = new LinkedHashSet<>();
for (String token : value.split(",")) {
advertised.add(token.trim());
}
return advertised;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import io.opensharing.asset.table.delta.DeltaKernel;
import io.opensharing.asset.table.delta.DeltaTableMetadataReader;
import io.opensharing.asset.table.delta.DeltaTableQueryReader;
import io.opensharing.asset.table.delta.SourceTableException;
import io.opensharing.auth.AuthContext;
import io.opensharing.auth.UserContext;
import io.opensharing.catalog.AssetLocation;
Expand All @@ -33,6 +34,7 @@
import java.util.Locale;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -124,8 +126,12 @@ public long version(
String startingTimestamp) {
ShareEntity share = protocolShares.requireGranted(recipient, shareName);
Instant timestamp = parseTimestamp("startingTimestamp", startingTimestamp);
return kernel.getVersion(
findTable(share, schemaAlias, tableAlias).catalogTable(), timestamp, owner(share));
ResolvedAsset table = findTable(share, schemaAlias, tableAlias).catalogTable();
return readingTable(
share,
schemaAlias,
tableAlias,
() -> kernel.getVersion(table, timestamp, owner(share)));
}

/**
Expand All @@ -142,9 +148,15 @@ public DeltaTableMetadataReader.Result metadata(
String timestamp,
String capabilities) {
ShareEntity share = protocolShares.requireGranted(recipient, shareName);
TableRequestChecks.requireAtMostOne("version,timestamp", version, timestamp);
TableRequestChecks.requireNotNegative("version", version);
Instant asOf = parseTimestamp("timestamp", timestamp);
ResolvedAsset table = findTable(share, schemaAlias, tableAlias).catalogTable();
return metadataReader.read(table, version, asOf, owner(share), capabilities);
return readingTable(
share,
schemaAlias,
tableAlias,
() -> metadataReader.read(table, version, asOf, owner(share), capabilities));
}

/**
Expand Down Expand Up @@ -202,12 +214,17 @@ public void query(
DeltaTableQueryReader.ResponseOptions options,
DeltaTableQueryReader.ResponseSink sink) {
ShareEntity share = protocolShares.requireGranted(recipient, shareName);
QueryTableRequest body = request == null ? QueryTableRequest.EMPTY : request;
requireValid(body);
ResolvedAsset catalogTable =
requireUrlAccess(
findTable(share, schemaAlias, tableAlias).catalogTable(), schemaAlias, tableAlias);
QueryTableRequest body = request == null ? QueryTableRequest.EMPTY : request;
if (body.startingVersion() != null || body.endingVersion() != null) {
queryChanges(body, catalogTable, share, options, sink);
readingTable(
share,
schemaAlias,
tableAlias,
() -> queryChanges(body, catalogTable, share, options, sink));
return;
}
boolean historical = body.version() != null || body.timestamp() != null;
Expand All @@ -223,17 +240,23 @@ public void query(
if (body.predicateHints() != null && !body.predicateHints().isEmpty()) {
log.debug("Ignoring deprecated predicateHints {}", body.predicateHints());
}
queryReader.read(
catalogTable,
owner(share),
options,
sink,
body.version(),
parseTimestamp("timestamp", body.timestamp()),
body.refreshToken(),
includeRefreshToken,
body.jsonPredicateHints(),
limitHint(body));
Instant timestamp = parseTimestamp("timestamp", body.timestamp());
readingTable(
share,
schemaAlias,
tableAlias,
() ->
queryReader.read(
catalogTable,
owner(share),
options,
sink,
body.version(),
timestamp,
body.refreshToken(),
includeRefreshToken,
body.jsonPredicateHints(),
limitHint(body)));
}

/**
Expand All @@ -251,20 +274,80 @@ public void changes(
DeltaTableQueryReader.ResponseOptions options,
DeltaTableQueryReader.ResponseSink sink) {
ShareEntity share = protocolShares.requireGranted(recipient, shareName);
requireValid(request);
ResolvedAsset catalogTable =
requireUrlAccess(
findTable(share, schemaAlias, tableAlias).catalogTable(), schemaAlias, tableAlias);
queryReader.readChangeDataFeed(
catalogTable,
owner(share),
options,
sink,
request.startingVersion(),
parseTimestamp("startingTimestamp", request.startingTimestamp()),
request.endingVersion(),
parseTimestamp("endingTimestamp", request.endingTimestamp()),
Boolean.TRUE.equals(request.includeHistoricalMetadata()),
Boolean.TRUE.equals(request.includeHistoricalProtocol()));
Instant startingTimestamp = parseTimestamp("startingTimestamp", request.startingTimestamp());
Instant endingTimestamp = parseTimestamp("endingTimestamp", request.endingTimestamp());
readingTable(
share,
schemaAlias,
tableAlias,
() ->
queryReader.readChangeDataFeed(
catalogTable,
owner(share),
options,
sink,
request.startingVersion(),
startingTimestamp,
request.endingVersion(),
endingTimestamp,
Boolean.TRUE.equals(request.includeHistoricalMetadata()),
Boolean.TRUE.equals(request.includeHistoricalProtocol())));
}

/** The version parameters of a snapshot or data change query, in the order they are checked.*/
private static void requireValid(QueryTableRequest body) {
TableRequestChecks.requireNotNegative("table version", body.version());
TableRequestChecks.requireNotNegative("startingVersion", body.startingVersion());
TableRequestChecks.requireNotNegative("endingVersion", body.endingVersion());
TableRequestChecks.requireAtMostOne(
"version,timestamp,startingVersion",
body.version(),
body.timestamp(),
body.startingVersion());
TableRequestChecks.requireOrderedVersions(body.startingVersion(), body.endingVersion());
}

/** The range parameters of a Change Data Feed query, in the order they are checked. */
private static void requireValid(TableChangesRequest request) {
TableRequestChecks.requireOneBoundary(
"starting", request.startingVersion(), request.startingTimestamp());
TableRequestChecks.requireOneBoundary(
"ending", request.endingVersion(), request.endingTimestamp());
TableRequestChecks.requireStart(request.startingVersion(), request.startingTimestamp());
TableRequestChecks.requireNotNegative("startingVersion", request.startingVersion());
TableRequestChecks.requireNotNegative("endingVersion", request.endingVersion());
if (request.startingVersion() != null && request.endingVersion() != null) {
TableRequestChecks.requireOrderedRange(request.startingVersion(), request.endingVersion());
}
}

/**
* Reads the table, answering a failure of the source table itself in the table's name as the
* share gives it, since the recipient knows no other.
*/
private <T> T readingTable(
ShareEntity share, String schemaAlias, String tableAlias, Supplier<T> read) {
try {
return read.get();
} catch (SourceTableException failed) {
throw failed.forRecipient(share.getName() + "." + schemaAlias + "." + tableAlias);
}
}

private void readingTable(
ShareEntity share, String schemaAlias, String tableAlias, Runnable read) {
readingTable(
share,
schemaAlias,
tableAlias,
() -> {
read.run();
return null;
});
}

/** Returns {@code catalogTable} once it is known to support the {@code url} access mode. */
Expand Down Expand Up @@ -293,21 +376,10 @@ private void queryChanges(
if (startingVersion == null) {
throw ApiException.invalidParameter("endingVersion requires startingVersion");
}
if (body.version() != null || body.timestamp() != null) {
throw ApiException.invalidParameter(
"Only one of version, timestamp, and startingVersion can be set");
}
if (Boolean.TRUE.equals(body.includeRefreshToken()) || body.refreshToken() != null) {
throw ApiException.invalidParameter(
"includeRefreshToken and refreshToken cannot be used with startingVersion");
}
if (startingVersion < 0) {
throw ApiException.invalidParameter("startingVersion must be non-negative");
}
if (endingVersion != null && endingVersion < startingVersion) {
throw ApiException.invalidParameter(
"endingVersion must be greater than or equal to startingVersion");
}
queryReader.readChanges(
table,
owner(share),
Expand Down
Loading
Loading