Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,13 @@ public class SharedDataObjectEntity extends BaseEntity {
@Column(name = "auxiliary_locations", length = 8192)
private String auxiliaryLocations;

/**
* The first version of a table that recipients may read, or null for all of them. A read that
* resolves to an earlier version is refused.
*/
@Column(name = "start_version")
private Long startVersion;

/** Lowercase schema name recipients see. */
@Column(name = "shared_as_schema", nullable = false, length = 255)
private String sharedAsSchema;
Expand Down Expand Up @@ -131,6 +138,14 @@ public void setAuxiliaryLocations(List<String> auxiliaryLocations) {
auxiliaryLocations.isEmpty() ? null : String.join("\n", auxiliaryLocations);
}

public Long getStartVersion() {
return startVersion;
}

public void setStartVersion(Long startVersion) {
this.startVersion = startVersion;
}

public String getSharedAsSchema() {
return sharedAsSchema;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ public SharedDataObjectEntity add(
AssetType type,
ResolvedAsset resolved,
String sharedAsSchema,
String sharedAsTable) {
String sharedAsTable,
Long startVersion) {
if (findByAlias(em, share, sharedAsSchema, sharedAsTable).isPresent()) {
throw ApiException.alreadyExists(
"alias '"
Expand Down Expand Up @@ -61,6 +62,7 @@ public SharedDataObjectEntity add(
}
object.setSharedAsSchema(sharedAsSchema);
object.setSharedAsTable(sharedAsTable);
object.setStartVersion(startVersion);
em.persist(object);
return object;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,9 @@ public DeltaTableMetadataReader.Result metadata(
String capabilities) {
ShareEntity share = protocolShares.requireGranted(recipient, shareName);
Instant asOf = parseTimestamp(timestamp);
ResolvedAsset table = findTable(share, schemaAlias, tableAlias).catalogTable();
return metadataReader.read(table, version, asOf, owner(share), capabilities);
SharedTable table = findTable(share, schemaAlias, tableAlias);
return metadataReader.read(
table.catalogTable(), table.startVersion(), version, asOf, owner(share), capabilities);
}

/**
Expand Down Expand Up @@ -203,9 +204,9 @@ public void query(
DeltaTableQueryReader.ResponseOptions options,
DeltaTableQueryReader.ResponseSink sink) {
ShareEntity share = protocolShares.requireGranted(recipient, shareName);
SharedTable sharedTable = findTable(share, schemaAlias, tableAlias);
ResolvedAsset catalogTable =
requireUrlAccess(
findTable(share, schemaAlias, tableAlias).catalogTable(), schemaAlias, tableAlias);
requireUrlAccess(sharedTable.catalogTable(), schemaAlias, tableAlias);
QueryTableRequest body = request == null ? QueryTableRequest.EMPTY : request;
int selectors =
(body.version() != null ? 1 : 0)
Expand All @@ -216,7 +217,7 @@ public void query(
"Please only provide one of: version,timestamp,startingVersion.");
}
if (body.startingVersion() != null || body.endingVersion() != null) {
queryChanges(body, catalogTable, share, options, sink);
queryChanges(body, catalogTable, sharedTable.startVersion(), share, options, sink);
return;
}
boolean historical = body.version() != null || body.timestamp() != null;
Expand All @@ -234,6 +235,7 @@ public void query(
}
queryReader.read(
catalogTable,
sharedTable.startVersion(),
owner(share),
options,
sink,
Expand All @@ -260,12 +262,13 @@ public void changes(
DeltaTableQueryReader.ResponseOptions options,
DeltaTableQueryReader.ResponseSink sink) {
ShareEntity share = protocolShares.requireGranted(recipient, shareName);
SharedTable sharedTable = findTable(share, schemaAlias, tableAlias);
ResolvedAsset catalogTable =
requireUrlAccess(
findTable(share, schemaAlias, tableAlias).catalogTable(), schemaAlias, tableAlias);
requireUrlAccess(sharedTable.catalogTable(), schemaAlias, tableAlias);
try {
queryReader.readChangeDataFeed(
catalogTable,
sharedTable.startVersion(),
owner(share),
options,
sink,
Expand Down Expand Up @@ -308,6 +311,7 @@ private static ResolvedAsset requireUrlAccess(
private void queryChanges(
QueryTableRequest body,
ResolvedAsset table,
long tableStartVersion,
ShareEntity share,
DeltaTableQueryReader.ResponseOptions options,
DeltaTableQueryReader.ResponseSink sink) {
Expand All @@ -333,6 +337,7 @@ private void queryChanges(
}
queryReader.readChanges(
table,
tableStartVersion,
owner(share),
options,
sink,
Expand All @@ -359,8 +364,13 @@ private SharedTable findTable(ShareEntity share, String schemaAlias, String tabl
if (sharedTable.isPresent()) {
ResolvedAsset catalogTable = resolveSharedAsset(share, sharedTable.get());
if (catalogTable != null) {
Long startVersion = sharedTable.get().getStartVersion();
return new SharedTable(
schemaAlias, tableAlias, sharedTable.get().getSourceAssetId(), catalogTable);
schemaAlias,
tableAlias,
sharedTable.get().getSourceAssetId(),
catalogTable,
startVersion == null ? 0 : startVersion);
}
}
// Recipients see aliases only, so the catalog's own not-found message must not reach them.
Expand Down Expand Up @@ -415,7 +425,11 @@ private static Instant parseTimestamp(String value) {

/** A table of the share, with the catalog table it resolved to. */
private record SharedTable(
String schemaAlias, String tableAlias, String tableId, ResolvedAsset catalogTable) {}
String schemaAlias,
String tableAlias,
String tableId,
ResolvedAsset catalogTable,
long startVersion) {}

/** The tables {@code sharedObjects} puts on the listing, by schema alias and table alias. */
private List<TableResponse> listTables(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,17 @@ final class DeltaSnapshots {

private DeltaSnapshots() {}

static Snapshot open(DeltaKernel.Session session, Long version, Instant timestamp) {
static Snapshot open(
DeltaKernel.Session session, Long version, Instant timestamp, long startVersion) {
Snapshot snapshot = openAt(session, version, timestamp);
if (snapshot.getVersion() < startVersion) {
throw ApiException.permissionDenied(
"You can only query table data since version " + startVersion + ".");
}
return snapshot;
}

private static Snapshot openAt(DeltaKernel.Session session, Long version, Instant timestamp) {
Snapshot latest = DeltaKernel.latestSnapshot(session.table(), session.engine());
if (version == null && timestamp == null) {
return latest;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ public DeltaTableMetadataReader(DeltaKernel kernel) {

public Result read(
ResolvedAsset table,
long startVersion,
Long version,
Instant timestamp,
AuthContext auth,
Expand All @@ -33,7 +34,7 @@ public Result read(
throw ApiException.invalidParameter("version and timestamp are mutually exclusive");
}
try (DeltaKernel.Session session = kernel.open(table, auth)) {
Snapshot snapshot = DeltaSnapshots.open(session, version, timestamp);
Snapshot snapshot = DeltaSnapshots.open(session, version, timestamp, startVersion);
SnapshotImpl impl = (SnapshotImpl) snapshot;
boolean historical = version != null || timestamp != null;
ResponseFormat format =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ public DeltaTableQueryReader(DeltaKernel kernel, UrlSigners signers) {
*/
public void read(
ResolvedAsset table,
long startVersion,
AuthContext auth,
ResponseOptions options,
ResponseSink sink,
Expand All @@ -140,7 +141,7 @@ public void read(

try (DeltaKernel.Session session = kernel.open(table, auth, URL_TTL);
ResponseWriter out = new ResponseWriter(session, table, options, sink)) {
Snapshot snapshot = DeltaSnapshots.open(session, version, timestamp);
Snapshot snapshot = DeltaSnapshots.open(session, version, timestamp, startVersion);
SnapshotImpl impl = (SnapshotImpl) snapshot;
DeltaSharingCapabilities.requireColumnMappingSupport(
options.capabilities(), impl.getMetadata().getConfiguration(), snapshot.getVersion());
Expand Down Expand Up @@ -196,22 +197,23 @@ public void read(
*/
public void readChanges(
ResolvedAsset table,
long startVersion,
AuthContext auth,
ResponseOptions options,
ResponseSink sink,
long startingVersion,
Long endingVersion,
boolean includeHistoricalProtocol) {
try (DeltaKernel.Session session = kernel.open(table, auth, URL_TTL)) {
long latest = DeltaSnapshots.open(session, null, null).getVersion();
long latest = DeltaSnapshots.open(session, null, null, startVersion).getVersion();
requireAtMostLatest("startingVersion", startingVersion, latest);
if (endingVersion != null) {
requireAtMostLatest("endingVersion", endingVersion, latest);
}
changes(
session,
table,
(SnapshotImpl) DeltaSnapshots.open(session, startingVersion, null),
(SnapshotImpl) DeltaSnapshots.open(session, startingVersion, null, startVersion),
ChangeRange.dataChanges(
startingVersion,
endingVersion == null ? latest : endingVersion,
Expand All @@ -230,6 +232,7 @@ public void readChanges(
*/
public void readChangeDataFeed(
ResolvedAsset table,
long startVersion,
AuthContext auth,
ResponseOptions options,
ResponseSink sink,
Expand All @@ -240,7 +243,7 @@ public void readChangeDataFeed(
boolean includeHistoricalMetadata,
boolean includeHistoricalProtocol) {
try (DeltaKernel.Session session = kernel.open(table, auth, URL_TTL)) {
SnapshotImpl latest = (SnapshotImpl) DeltaSnapshots.open(session, null, null);
SnapshotImpl latest = (SnapshotImpl) DeltaSnapshots.open(session, null, null, startVersion);
TableImpl delta = (TableImpl) session.table();
if (endingTimestamp != null) {
requireEndingTimestampAtMostLatest(endingTimestamp, latest, session);
Expand Down Expand Up @@ -268,7 +271,7 @@ public void readChangeDataFeed(
"endingVersion " + end + " is before startingVersion " + start);
}
long resolvedEnd = end == null ? latest.getVersion() : Math.min(end, latest.getVersion());
SnapshotImpl first = (SnapshotImpl) DeltaSnapshots.open(session, start, null);
SnapshotImpl first = (SnapshotImpl) DeltaSnapshots.open(session, start, null, startVersion);
if (!changeDataFeedEnabled(first.getMetadata())) {
throw new ChangeDataNotRecordedException(start);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
* "displayName": "<display name>",
* "comment": "<comment>",
* "properties": {"<key>": "<value>"},
* "objects": [{"name": "<asset name>", "type": "TABLE", "sharedAs": "<alias>"}],
* "objects": [{"name": "<asset name>", "type": "TABLE", "sharedAs": "<alias>",
* "startVersion": 1}],
* "ownerId": "<owner user id>"
* }
* }</pre>
Expand Down Expand Up @@ -55,11 +56,16 @@ public static ShareResponse from(ShareEntity share, List<SharedDataObjectEntity>
}

/** A catalog asset included in a share, with the alias recipients see it under. */
public record SharedDataObjectResponse(String name, AssetType type, String sharedAs) {
@JsonInclude(JsonInclude.Include.NON_NULL)
public record SharedDataObjectResponse(
String name, AssetType type, String sharedAs, Long startVersion) {

static SharedDataObjectResponse from(SharedDataObjectEntity object) {
return new SharedDataObjectResponse(
object.getFullName(), object.getType(), object.getSharedAs());
object.getFullName(),
object.getType(),
object.getSharedAs(),
object.getStartVersion());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,8 @@ public ShareResponse update(UserContext user, String share, UpdateShareRequest r
addition.type(),
addition.resolved(),
addition.alias().schema(),
addition.alias().table());
addition.alias().table(),
addition.startVersion());
}
shares.update(entity, displayName, comment, properties, newOwnerId);
return entity;
Expand Down Expand Up @@ -302,7 +303,10 @@ private Addition addition(UserContext user, UpdateShareRequest.DataObject dataOb
throw new CatalogException(
"catalog resolved '" + name + "' as " + resolved.type() + " instead of " + type);
}
return new Addition(name, type, resolved, alias);
if (dataObject.startVersion() != null && dataObject.startVersion() < 0) {
throw ApiException.invalidParameter("dataObject.startVersion must not be negative");
}
return new Addition(name, type, resolved, alias, dataObject.startVersion());
}

// Removes by alias when sharedAs is given, otherwise by catalog name.
Expand Down Expand Up @@ -391,7 +395,8 @@ private static String requireMaxLength(String value, int maxLength, String field

private record Alias(String schema, String table) {}

private record Addition(String name, AssetType type, ResolvedAsset resolved, Alias alias) {}
private record Addition(
String name, AssetType type, ResolvedAsset resolved, Alias alias, Long startVersion) {}

private record RecipientChange(
RecipientEntity recipient, UpdateSharePermissionsRequest.Change change) {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,10 @@ public record Update(Action action, DataObject dataObject) {}
/**
* A catalog table. {@code name} is the full catalog name; {@code sharedAs} is the alias
* recipients see, {@code schema.table}, and defaults to {@code name} without its catalog. A
* remove needs {@code name} or {@code sharedAs}.
* remove needs {@code name} or {@code sharedAs}. {@code startVersion}, if set, is the first
* version of the table that recipients may read.
*/
public record DataObject(String name, AssetType type, String sharedAs) {}
public record DataObject(String name, AssetType type, String sharedAs, Long startVersion) {}

public enum Action {
ADD,
Expand Down
Loading
Loading