diff --git a/README.md b/README.md index a0803ec90..79540dfcd 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,8 @@ without putting the raw address in the log. | storage.bucket | - | No | Blob storage bucket. | | storage.overrides.* | - | No | Key-value pairs to override storage settings. `*` might be any specific blob storage setting to be overridden. Refer to [examples](#temporary-credentials-1) in the sections below. | | storage.createBucket | false | No | Indicates whether bucket should be created on start-up. | +| storageLayout.tenantRooted | false | No | Places every bucket under a tenant root instead of storing it at the top level. Changing it on a populated deployment re-addresses existing data and requires migration; leave disabled otherwise. | +| storageLayout.defaultTenant | default | No | Tenant that existing buckets are placed under when `storageLayout.tenantRooted` is enabled. | | storage.prefix | - | No | Base prefix for all stored resources. The purpose to use the same bucket for different environments, e.g. dev, prod, pre-prod. Must not contain path separators or any invalid chars. | | storage.maxUploadedFileSize | 536870912 | No | Maximum size in bytes of uploaded file. If a size of uploaded file exceeds the limit the server returns HTTP code 413 | diff --git a/server/src/main/java/com/epam/aidial/core/server/AiDial.java b/server/src/main/java/com/epam/aidial/core/server/AiDial.java index 3721acb2e..3677a3fe3 100644 --- a/server/src/main/java/com/epam/aidial/core/server/AiDial.java +++ b/server/src/main/java/com/epam/aidial/core/server/AiDial.java @@ -87,8 +87,12 @@ import com.epam.aidial.core.storage.blobstore.BlobStorage; import com.epam.aidial.core.storage.blobstore.Storage; import com.epam.aidial.core.storage.cache.CacheClientFactory; +import com.epam.aidial.core.storage.resource.LegacyStorageLayout; import com.epam.aidial.core.storage.resource.ResourceDescriptor; import com.epam.aidial.core.storage.resource.ResourceTypes; +import com.epam.aidial.core.storage.resource.StorageLayout; +import com.epam.aidial.core.storage.resource.StorageLayouts; +import com.epam.aidial.core.storage.resource.TenantRootedStorageLayout; import com.epam.aidial.core.storage.service.LockService; import com.epam.aidial.core.storage.service.ResourceService; import com.epam.aidial.core.storage.service.TimerService; @@ -202,6 +206,8 @@ void start() throws Exception { accessTokenValidator = new AccessTokenValidator(settings("identityProviders"), vertx, taskExecutor, client, clientOptions, claimsLogLevel); } + StorageLayouts.useLayout(createStorageLayout(settings("storageLayout"))); + if (storage == null) { Storage storageConfig = Json.decodeValue(settings("storage").toBuffer(), Storage.class); storage = new BlobStorage(storageConfig); @@ -489,6 +495,14 @@ private JsonObject settings(String key) { return settings.getJsonObject(key, new JsonObject()); } + private static StorageLayout createStorageLayout(JsonObject settings) { + if (!settings.getBoolean("tenantRooted", false)) { + return LegacyStorageLayout.INSTANCE; + } + + return new TenantRootedStorageLayout(settings.getString("defaultTenant", "default")); + } + private List getAllowedRedirectUris() { return settings("toolsets") .getJsonObject("security", new JsonObject()) diff --git a/server/src/main/resources/aidial.settings.json b/server/src/main/resources/aidial.settings.json index f65f4bb57..db960d2b3 100644 --- a/server/src/main/resources/aidial.settings.json +++ b/server/src/main/resources/aidial.settings.json @@ -55,6 +55,10 @@ "jclouds.filesystem.basedir": "data" } }, + "storageLayout": { + "tenantRooted": false, + "defaultTenant": "default" + }, "resources": { "maxSize" : 67108864, "maxSizeToCache": 1048576, diff --git a/storage/src/main/java/com/epam/aidial/core/storage/resource/ResourceDescriptor.java b/storage/src/main/java/com/epam/aidial/core/storage/resource/ResourceDescriptor.java index 41de2fc54..caa6a8c98 100644 --- a/storage/src/main/java/com/epam/aidial/core/storage/resource/ResourceDescriptor.java +++ b/storage/src/main/java/com/epam/aidial/core/storage/resource/ResourceDescriptor.java @@ -23,10 +23,6 @@ public class ResourceDescriptor { public static final String PLATFORM_BUCKET = "platform"; public static final String PLATFORM_LOCATION = PLATFORM_BUCKET + PATH_SEPARATOR; - /** - * Layout in force for physical paths. Swapped once the tenant-rooted layout is enabled. - */ - private static final StorageLayout LAYOUT = LegacyStorageLayout.INSTANCE; ResourceType type; /** @@ -134,7 +130,8 @@ public String getAbsoluteFilePath() { * location followed by the resource-type folder. */ private String getStoragePrefix() { - return LAYOUT.resolveLocationPrefix(bucketLocation) + LAYOUT.resolveTypeFolder(type.group()) + PATH_SEPARATOR; + StorageLayout layout = StorageLayouts.resolveActive(); + return layout.resolveLocationPrefix(bucketLocation) + layout.resolveTypeFolder(type.group()) + PATH_SEPARATOR; } /** diff --git a/storage/src/main/java/com/epam/aidial/core/storage/resource/StorageLayouts.java b/storage/src/main/java/com/epam/aidial/core/storage/resource/StorageLayouts.java new file mode 100644 index 000000000..cb3eb6e22 --- /dev/null +++ b/storage/src/main/java/com/epam/aidial/core/storage/resource/StorageLayouts.java @@ -0,0 +1,22 @@ +package com.epam.aidial.core.storage.resource; + +/** + * Holds the layout every physical path is composed with. Set once during start-up, before any resource + * is read or written; process-wide because {@link ResourceDescriptor} is constructed everywhere and + * carries no configuration of its own. + */ +public final class StorageLayouts { + + private static volatile StorageLayout active = LegacyStorageLayout.INSTANCE; + + private StorageLayouts() { + } + + public static StorageLayout resolveActive() { + return active; + } + + public static void useLayout(StorageLayout layout) { + active = layout; + } +} diff --git a/storage/src/main/java/com/epam/aidial/core/storage/resource/TenantRootedStorageLayout.java b/storage/src/main/java/com/epam/aidial/core/storage/resource/TenantRootedStorageLayout.java new file mode 100644 index 000000000..ffabeffce --- /dev/null +++ b/storage/src/main/java/com/epam/aidial/core/storage/resource/TenantRootedStorageLayout.java @@ -0,0 +1,28 @@ +package com.epam.aidial.core.storage.resource; + +/** + * The tenant-rooted layout: every bucket location is placed under its tenant, and resource-type folders + * are reserved names. Conversion rules live in {@link TenantLayoutTransform}. + */ +public final class TenantRootedStorageLayout implements StorageLayout { + + private final String tenantId; + + public TenantRootedStorageLayout(String tenantId) { + if (tenantId == null || tenantId.isBlank()) { + throw new IllegalArgumentException("Tenant id must not be blank"); + } + + this.tenantId = tenantId; + } + + @Override + public String resolveLocationPrefix(String bucketLocation) { + return TenantLayoutTransform.toTenantLocation(bucketLocation, tenantId); + } + + @Override + public String resolveTypeFolder(String group) { + return TenantLayoutTransform.toTenantTypeFolder(group); + } +} diff --git a/storage/src/test/java/com/epam/aidial/core/storage/resource/StorageLayoutsTest.java b/storage/src/test/java/com/epam/aidial/core/storage/resource/StorageLayoutsTest.java new file mode 100644 index 000000000..53cad8f11 --- /dev/null +++ b/storage/src/test/java/com/epam/aidial/core/storage/resource/StorageLayoutsTest.java @@ -0,0 +1,42 @@ +package com.epam.aidial.core.storage.resource; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +public class StorageLayoutsTest { + + @AfterEach + public void restoreDefaultLayout() { + StorageLayouts.useLayout(LegacyStorageLayout.INSTANCE); + } + + @Test + public void testLegacyLayoutIsActiveByDefault() { + assertSame(LegacyStorageLayout.INSTANCE, StorageLayouts.resolveActive()); + } + + @Test + public void testActiveLayoutIsReplaceable() { + StorageLayout tenantRooted = new TenantRootedStorageLayout("acme"); + StorageLayouts.useLayout(tenantRooted); + + assertSame(tenantRooted, StorageLayouts.resolveActive()); + } + + @Test + public void testDescriptorPathFollowsActiveLayout() { + ResourceDescriptor file = new ResourceDescriptor(ResourceTypes.FILE, "notes.txt", + List.of("documents"), "bucket", "Users/u1/", false); + + assertEquals("Users/u1/files/documents/notes.txt", file.getAbsoluteFilePath()); + + StorageLayouts.useLayout(new TenantRootedStorageLayout("acme")); + + assertEquals(".org/acme/.users/u1/.files/documents/notes.txt", file.getAbsoluteFilePath()); + } +} diff --git a/storage/src/test/java/com/epam/aidial/core/storage/resource/TenantRootedStorageLayoutTest.java b/storage/src/test/java/com/epam/aidial/core/storage/resource/TenantRootedStorageLayoutTest.java new file mode 100644 index 000000000..bc87cab9a --- /dev/null +++ b/storage/src/test/java/com/epam/aidial/core/storage/resource/TenantRootedStorageLayoutTest.java @@ -0,0 +1,36 @@ +package com.epam.aidial.core.storage.resource; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class TenantRootedStorageLayoutTest { + + private final StorageLayout layout = new TenantRootedStorageLayout("acme"); + + @Test + public void testLocationPrefixIsTenantRooted() { + assertEquals(".org/acme/.users/u1/", layout.resolveLocationPrefix("Users/u1/")); + assertEquals(".org/acme/.keys/proj/", layout.resolveLocationPrefix("Keys/proj/")); + assertEquals(".org/acme/", layout.resolveLocationPrefix("public/")); + assertEquals("", layout.resolveLocationPrefix("platform/")); + } + + @Test + public void testTypeFolderIsReserved() { + assertEquals(".files", layout.resolveTypeFolder("files")); + assertEquals(".conversations", layout.resolveTypeFolder("conversations")); + } + + @Test + public void testUnsupportedLocationRejected() { + assertThrows(IllegalArgumentException.class, () -> layout.resolveLocationPrefix("Unknown/u1/")); + } + + @Test + public void testBlankTenantRejected() { + assertThrows(IllegalArgumentException.class, () -> new TenantRootedStorageLayout(null)); + assertThrows(IllegalArgumentException.class, () -> new TenantRootedStorageLayout(" ")); + } +}