diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/StorageLocations.java b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/StorageLocations.java index 3892f5ff8..4e6cc7e8e 100644 --- a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/StorageLocations.java +++ b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/StorageLocations.java @@ -12,8 +12,11 @@ public final class StorageLocations { public static final Set S3_SCHEMES = Set.of("s3", "s3a", "s3n"); - /** Azure file URLs are signed only for abfs:// and abfss://, so WASB tables are not read. */ - public static final Set AZURE_SCHEMES = Set.of("abfs", "abfss"); + /** + * WASB locations are read through ABFS at the same blob host, and their file URLs are signed on + * that host, so all four Azure schemes are supported. + */ + public static final Set AZURE_SCHEMES = Set.of("abfs", "abfss", "wasb", "wasbs"); public static final Set GCS_SCHEMES = Set.of("gs"); diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/HadoopStorageConfiguration.java b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/HadoopStorageConfiguration.java index 937402d06..56770dac0 100644 --- a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/HadoopStorageConfiguration.java +++ b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/HadoopStorageConfiguration.java @@ -55,7 +55,8 @@ static Configuration from(StorageCredentials credentials, String location) { /** * Hadoop reads S3 and R2 only through its s3a:// connector. Catalogs typically return S3 - * locations as s3://, and Hadoop no longer reads s3n://. + * locations as s3://, and Hadoop no longer reads s3n://. Hadoop reads WASB locations through + * ABFS at the same blob host. */ static String kernelPath(String location) { String scheme = StorageLocations.scheme(location); @@ -66,6 +67,9 @@ static String kernelPath(String location) { StorageLocations.R2Object r2 = StorageLocations.r2Object(location); return "s3a://" + r2.bucket() + "/" + r2.key(); } + if (StorageLocations.scheme(location).startsWith("wasb")) { + return "abfss://" + location.substring(location.indexOf("://") + 3); + } return location; } @@ -105,7 +109,10 @@ private static void configureR2( configuration.set("fs.s3a.path.style.access", "true"); } - /** The catalog's SAS token for every request to the storage account. */ + /** + * The catalog's SAS token for every request to the storage account. ABFS reads a blob host, which + * WASB locations name, only for an account declared to have no hierarchical namespace. + */ private static void configureAzure( Configuration configuration, StorageCredentials credentials, String location) { String host = URI.create(location).getHost(); @@ -115,6 +122,9 @@ private static void configureAzure( configuration.set("fs.azure.account.auth.type." + host, "SAS"); configuration.set( "fs.azure.sas.fixed.token." + host, credentials.require(StorageCredentials.SAS_TOKEN)); + if (host.contains(".blob.")) { + configuration.set("fs.azure.account.hns.enabled." + host, "false"); + } } /** diff --git a/server/opensharing-server-core/src/test/java/io/opensharing/asset/table/delta/HadoopStorageConfigurationTest.java b/server/opensharing-server-core/src/test/java/io/opensharing/asset/table/delta/HadoopStorageConfigurationTest.java index 51c60500b..c91e1b8c7 100644 --- a/server/opensharing-server-core/src/test/java/io/opensharing/asset/table/delta/HadoopStorageConfigurationTest.java +++ b/server/opensharing-server-core/src/test/java/io/opensharing/asset/table/delta/HadoopStorageConfigurationTest.java @@ -49,11 +49,13 @@ void readsOnlySupportedStorage() { "gs://bucket/table", "abfs://container@account.dfs.core.windows.net/table", "abfss://container@account.dfs.core.windows.net/table", + "wasb://container@account.blob.core.windows.net/table", + "wasbs://container@account.blob.core.windows.net/table", "file:///tmp/table", "/tmp/table")) { assertDoesNotThrow(() -> HadoopStorageConfiguration.requireSupported(location), location); } - for (String scheme : List.of("wasb", "wasbs", "hdfs")) { + for (String scheme : List.of("hdfs")) { ApiException rejected = assertThrows( ApiException.class, @@ -64,6 +66,25 @@ void readsOnlySupportedStorage() { } } + @Test + void readsWasbThroughAbfsAtTheBlobHost() throws IOException { + String location = "wasbs://container@account.blob.core.windows.net/table"; + String host = "account.blob.core.windows.net"; + Configuration configuration = + HadoopStorageConfiguration.from(sasCredentials(location), location); + + assertEquals( + "abfss://container@" + host + "/table", HadoopStorageConfiguration.kernelPath(location)); + assertEquals( + "abfss://container@" + host + "/table", + HadoopStorageConfiguration.kernelPath("wasb://container@" + host + "/table")); + assertEquals("SAS", configuration.get("fs.azure.account.auth.type." + host)); + assertEquals( + "sv=2020-12-06&sig=catalog", configuration.get("fs.azure.sas.fixed.token." + host)); + assertEquals("false", configuration.get("fs.azure.account.hns.enabled." + host)); + assertOpens("org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem", location, configuration); + } + @Test void readsEveryS3SchemeThroughS3A() throws IOException { Configuration configuration = new Configuration(); @@ -104,4 +125,12 @@ private static StorageCredentials r2Credentials(String location) { StorageCredentials.SESSION_TOKEN, "session"), Instant.now().plus(Duration.ofHours(1))); } + + private static StorageCredentials sasCredentials(String location) { + return new StorageCredentials( + location, + CloudProvider.AZURE, + Map.of(StorageCredentials.SAS_TOKEN, "sv=2020-12-06&sig=catalog"), + Instant.now().plus(Duration.ofHours(1))); + } } diff --git a/server/opensharing-server-core/src/test/java/io/opensharing/asset/table/signer/UrlSignersTest.java b/server/opensharing-server-core/src/test/java/io/opensharing/asset/table/signer/UrlSignersTest.java index ed0042bd4..577d649ea 100644 --- a/server/opensharing-server-core/src/test/java/io/opensharing/asset/table/signer/UrlSignersTest.java +++ b/server/opensharing-server-core/src/test/java/io/opensharing/asset/table/signer/UrlSignersTest.java @@ -73,7 +73,7 @@ void rejectsUnknownSchemesAndExpiredCredentials() { } @Test - void signsAzurePathsOnlyInAbfsForm() { + void signsAzurePathsAtTheBlobHost() { StorageCredentials sas = new StorageCredentials( "abfss://container@account.dfs.core.windows.net/", @@ -92,17 +92,15 @@ void signsAzurePathsOnlyInAbfsForm() { signed.url()); } for (String scheme : List.of("wasb", "wasbs")) { - ApiException rejected = - assertThrows( - ApiException.class, - () -> - sign( - scheme + "://container@account.blob.core.windows.net/table/part.parquet", - sas, - Duration.ofMinutes(5))); - assertEquals(501, rejected.getStatus()); - assertEquals( - "this build cannot sign urls for '" + scheme + "' storage", rejected.getMessage()); + SignedUrl signed = + sign( + scheme + "://container@account.blob.core.windows.net/table/part.parquet", + sas, + Duration.ofMinutes(5)); + assertTrue( + signed.url().startsWith("https://account.blob.core.windows.net/container/table/"), + signed.url()); + assertTrue(signed.url().endsWith("sv=2020-12-06&sig=catalog"), signed.url()); } } diff --git a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolUnsupportedStorageTest.java b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolUnsupportedStorageTest.java index 8a303b897..03569584a 100644 --- a/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolUnsupportedStorageTest.java +++ b/server/opensharing-server/src/test/java/io/opensharing/protocol/ProtocolUnsupportedStorageTest.java @@ -7,13 +7,12 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; import java.util.Properties; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -/** Reads of tables in WASB storage, which this build does not read, through the local catalog. */ +/** Reads of tables in storage this build does not read, through the local catalog. */ class ProtocolUnsupportedStorageTest extends ProtocolApiSupport { @TempDir static Path directory; @@ -32,15 +31,10 @@ static void startUnsupportedStorageServer() throws IOException { assets: - identifier: main.lake type: SCHEMA - - identifier: main.lake.wasb + - identifier: main.lake.hdfs attributes: subtype: MANAGED - storageLocation: wasb://container@account.blob.core.windows.net/wasb - format: delta - - identifier: main.lake.wasbs - attributes: - subtype: MANAGED - storageLocation: wasbs://container@account.blob.core.windows.net/wasbs + storageLocation: hdfs://namenode:8020/table format: delta """; Path file = Files.writeString(directory.resolve("local-catalog.yml"), catalog); @@ -52,21 +46,17 @@ static void startUnsupportedStorageServer() throws IOException { @Test void rejectsReadsBeforeTheResponseStarts() { createShare("unsupported-storage"); - for (String table : List.of("wasb", "wasbs")) { - addObject("unsupported-storage", "TABLE", "main.lake." + table, "lake." + table); - } + addObject("unsupported-storage", "TABLE", "main.lake.hdfs", "lake.hdfs"); String bearer = createAndActivateRecipient("unsupported-storage-partner"); grant("unsupported-storage", "unsupported-storage-partner"); - String tables = PROTOCOL + "/shares/unsupported-storage/schemas/lake/tables/"; - for (String table : List.of("wasb", "wasbs")) { - String message = "this build cannot read tables in '" + table + "' storage"; - assertNotImplemented(message, server.get(tables + table + "/version", bearer)); - assertNotImplemented(message, server.get(tables + table + "/metadata", bearer)); - assertNotImplemented(message, server.post(tables + table + "/query", bearer, "{}")); - assertNotImplemented( - message, server.get(tables + table + "/changes?startingVersion=0", bearer)); - } + String tables = PROTOCOL + "/shares/unsupported-storage/schemas/lake/tables/hdfs"; + String message = "this build cannot read tables in 'hdfs' storage"; + assertNotImplemented(message, server.get(tables + "/version", bearer)); + assertNotImplemented(message, server.get(tables + "/metadata", bearer)); + assertNotImplemented(message, server.post(tables + "/query", bearer, "{}")); + assertNotImplemented( + message, server.get(tables + "/changes?startingVersion=0", bearer)); } private static void assertNotImplemented(String message, Response response) {