Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,11 @@ public final class StorageLocations {

public static final Set<String> S3_SCHEMES = Set.of("s3", "s3a", "s3n");

/** Azure file URLs are signed only for abfs:// and abfss://, so WASB tables are not read. */
public static final Set<String> AZURE_SCHEMES = Set.of("abfs", "abfss");
/**
* WASB locations are read through ABFS at the same blob host, and their file URLs are signed on
* that host, so all four Azure schemes are supported.
*/
public static final Set<String> AZURE_SCHEMES = Set.of("abfs", "abfss", "wasb", "wasbs");

public static final Set<String> GCS_SCHEMES = Set.of("gs");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ static Configuration from(StorageCredentials credentials, String location) {

/**
* Hadoop reads S3 and R2 only through its s3a:// connector. Catalogs typically return S3
* locations as s3://, and Hadoop no longer reads s3n://.
* locations as s3://, and Hadoop no longer reads s3n://. Hadoop reads WASB locations through
* ABFS at the same blob host.
*/
static String kernelPath(String location) {
String scheme = StorageLocations.scheme(location);
Expand All @@ -66,6 +67,9 @@ static String kernelPath(String location) {
StorageLocations.R2Object r2 = StorageLocations.r2Object(location);
return "s3a://" + r2.bucket() + "/" + r2.key();
}
if (StorageLocations.scheme(location).startsWith("wasb")) {
return "abfss://" + location.substring(location.indexOf("://") + 3);
}
return location;
}

Expand Down Expand Up @@ -105,7 +109,10 @@ private static void configureR2(
configuration.set("fs.s3a.path.style.access", "true");
}

/** The catalog's SAS token for every request to the storage account. */
/**
* The catalog's SAS token for every request to the storage account. ABFS reads a blob host, which
* WASB locations name, only for an account declared to have no hierarchical namespace.
*/
private static void configureAzure(
Configuration configuration, StorageCredentials credentials, String location) {
String host = URI.create(location).getHost();
Expand All @@ -115,6 +122,9 @@ private static void configureAzure(
configuration.set("fs.azure.account.auth.type." + host, "SAS");
configuration.set(
"fs.azure.sas.fixed.token." + host, credentials.require(StorageCredentials.SAS_TOKEN));
if (host.contains(".blob.")) {
configuration.set("fs.azure.account.hns.enabled." + host, "false");
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,13 @@ void readsOnlySupportedStorage() {
"gs://bucket/table",
"abfs://container@account.dfs.core.windows.net/table",
"abfss://container@account.dfs.core.windows.net/table",
"wasb://container@account.blob.core.windows.net/table",
"wasbs://container@account.blob.core.windows.net/table",
"file:///tmp/table",
"/tmp/table")) {
assertDoesNotThrow(() -> HadoopStorageConfiguration.requireSupported(location), location);
}
for (String scheme : List.of("wasb", "wasbs", "hdfs")) {
for (String scheme : List.of("hdfs")) {
ApiException rejected =
assertThrows(
ApiException.class,
Expand All @@ -64,6 +66,25 @@ void readsOnlySupportedStorage() {
}
}

@Test
void readsWasbThroughAbfsAtTheBlobHost() throws IOException {
String location = "wasbs://container@account.blob.core.windows.net/table";
String host = "account.blob.core.windows.net";
Configuration configuration =
HadoopStorageConfiguration.from(sasCredentials(location), location);

assertEquals(
"abfss://container@" + host + "/table", HadoopStorageConfiguration.kernelPath(location));
assertEquals(
"abfss://container@" + host + "/table",
HadoopStorageConfiguration.kernelPath("wasb://container@" + host + "/table"));
assertEquals("SAS", configuration.get("fs.azure.account.auth.type." + host));
assertEquals(
"sv=2020-12-06&sig=catalog", configuration.get("fs.azure.sas.fixed.token." + host));
assertEquals("false", configuration.get("fs.azure.account.hns.enabled." + host));
assertOpens("org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem", location, configuration);
}

@Test
void readsEveryS3SchemeThroughS3A() throws IOException {
Configuration configuration = new Configuration();
Expand Down Expand Up @@ -104,4 +125,12 @@ private static StorageCredentials r2Credentials(String location) {
StorageCredentials.SESSION_TOKEN, "session"),
Instant.now().plus(Duration.ofHours(1)));
}

private static StorageCredentials sasCredentials(String location) {
return new StorageCredentials(
location,
CloudProvider.AZURE,
Map.of(StorageCredentials.SAS_TOKEN, "sv=2020-12-06&sig=catalog"),
Instant.now().plus(Duration.ofHours(1)));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ void rejectsUnknownSchemesAndExpiredCredentials() {
}

@Test
void signsAzurePathsOnlyInAbfsForm() {
void signsAzurePathsAtTheBlobHost() {
StorageCredentials sas =
new StorageCredentials(
"abfss://container@account.dfs.core.windows.net/",
Expand All @@ -92,17 +92,15 @@ void signsAzurePathsOnlyInAbfsForm() {
signed.url());
}
for (String scheme : List.of("wasb", "wasbs")) {
ApiException rejected =
assertThrows(
ApiException.class,
() ->
sign(
scheme + "://container@account.blob.core.windows.net/table/part.parquet",
sas,
Duration.ofMinutes(5)));
assertEquals(501, rejected.getStatus());
assertEquals(
"this build cannot sign urls for '" + scheme + "' storage", rejected.getMessage());
SignedUrl signed =
sign(
scheme + "://container@account.blob.core.windows.net/table/part.parquet",
sas,
Duration.ofMinutes(5));
assertTrue(
signed.url().startsWith("https://account.blob.core.windows.net/container/table/"),
signed.url());
assertTrue(signed.url().endsWith("sv=2020-12-06&sig=catalog"), signed.url());
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,12 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Properties;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

/** Reads of tables in WASB storage, which this build does not read, through the local catalog. */
/** Reads of tables in storage this build does not read, through the local catalog. */
class ProtocolUnsupportedStorageTest extends ProtocolApiSupport {

@TempDir static Path directory;
Expand All @@ -32,15 +31,10 @@ static void startUnsupportedStorageServer() throws IOException {
assets:
- identifier: main.lake
type: SCHEMA
- identifier: main.lake.wasb
- identifier: main.lake.hdfs
attributes:
subtype: MANAGED
storageLocation: wasb://container@account.blob.core.windows.net/wasb
format: delta
- identifier: main.lake.wasbs
attributes:
subtype: MANAGED
storageLocation: wasbs://container@account.blob.core.windows.net/wasbs
storageLocation: hdfs://namenode:8020/table
format: delta
""";
Path file = Files.writeString(directory.resolve("local-catalog.yml"), catalog);
Expand All @@ -52,21 +46,17 @@ static void startUnsupportedStorageServer() throws IOException {
@Test
void rejectsReadsBeforeTheResponseStarts() {
createShare("unsupported-storage");
for (String table : List.of("wasb", "wasbs")) {
addObject("unsupported-storage", "TABLE", "main.lake." + table, "lake." + table);
}
addObject("unsupported-storage", "TABLE", "main.lake.hdfs", "lake.hdfs");
String bearer = createAndActivateRecipient("unsupported-storage-partner");
grant("unsupported-storage", "unsupported-storage-partner");

String tables = PROTOCOL + "/shares/unsupported-storage/schemas/lake/tables/";
for (String table : List.of("wasb", "wasbs")) {
String message = "this build cannot read tables in '" + table + "' storage";
assertNotImplemented(message, server.get(tables + table + "/version", bearer));
assertNotImplemented(message, server.get(tables + table + "/metadata", bearer));
assertNotImplemented(message, server.post(tables + table + "/query", bearer, "{}"));
assertNotImplemented(
message, server.get(tables + table + "/changes?startingVersion=0", bearer));
}
String tables = PROTOCOL + "/shares/unsupported-storage/schemas/lake/tables/hdfs";
String message = "this build cannot read tables in 'hdfs' storage";
assertNotImplemented(message, server.get(tables + "/version", bearer));
assertNotImplemented(message, server.get(tables + "/metadata", bearer));
assertNotImplemented(message, server.post(tables + "/query", bearer, "{}"));
assertNotImplemented(
message, server.get(tables + "/changes?startingVersion=0", bearer));
}

private static void assertNotImplemented(String message, Response response) {
Expand Down
Loading