feat: add tenant-rooted storage layout behind a setting #1863

README.md

@@ -232,6 +232,8 @@ without putting the raw address in the log.
 | storage.bucket              |     -      |    No    | Blob storage bucket.                                                                                                                                                                               |
 | storage.overrides.*         |     -      |    No    | Key-value pairs to override storage settings. `*` might be any specific blob storage setting to be overridden. Refer to [examples](#temporary-credentials-1) in the sections below.                |
 | storage.createBucket        |   false    |    No    | Indicates whether bucket should be created on start-up.                                                                                                                                            |
+| storageLayout.tenantRooted  |   false    |    No    | Places every bucket under a tenant root instead of storing it at the top level. Changing it on a populated deployment re-addresses existing data and requires migration; leave disabled otherwise.  |
+| storageLayout.defaultTenant |  default   |    No    | Tenant that existing buckets are placed under when `storageLayout.tenantRooted` is enabled.                                                                                                         |
 | storage.prefix              |     -      |    No    | Base prefix for all stored resources. The purpose to use the same bucket for different environments, e.g. dev, prod, pre-prod. Must not contain path separators or any invalid chars.              |
 | storage.maxUploadedFileSize | 536870912  |    No    | Maximum size in bytes of uploaded file. If a size of uploaded file exceeds the limit the server returns HTTP code 413                                                                              |
 

server/src/main/java/com/epam/aidial/core/server/AiDial.java

@@ -87,8 +87,12 @@
 import com.epam.aidial.core.storage.blobstore.BlobStorage;
 import com.epam.aidial.core.storage.blobstore.Storage;
 import com.epam.aidial.core.storage.cache.CacheClientFactory;
+import com.epam.aidial.core.storage.resource.LegacyStorageLayout;
 import com.epam.aidial.core.storage.resource.ResourceDescriptor;
 import com.epam.aidial.core.storage.resource.ResourceTypes;
+import com.epam.aidial.core.storage.resource.StorageLayout;
+import com.epam.aidial.core.storage.resource.StorageLayouts;
+import com.epam.aidial.core.storage.resource.TenantRootedStorageLayout;
 import com.epam.aidial.core.storage.service.LockService;
 import com.epam.aidial.core.storage.service.ResourceService;
 import com.epam.aidial.core.storage.service.TimerService;
@@ -202,6 +206,8 @@ void start() throws Exception {
                 accessTokenValidator = new AccessTokenValidator(settings("identityProviders"), vertx, taskExecutor, client, clientOptions, claimsLogLevel);
             }
 
+            StorageLayouts.useLayout(createStorageLayout(settings("storageLayout")));
+
             if (storage == null) {
                 Storage storageConfig = Json.decodeValue(settings("storage").toBuffer(), Storage.class);
                 storage = new BlobStorage(storageConfig);
@@ -489,6 +495,14 @@ private JsonObject settings(String key) {
         return settings.getJsonObject(key, new JsonObject());
     }
 
+    private static StorageLayout createStorageLayout(JsonObject settings) {
+        if (!settings.getBoolean("tenantRooted", false)) {
+            return LegacyStorageLayout.INSTANCE;
+        }
+
+        return new TenantRootedStorageLayout(settings.getString("defaultTenant", "default"));
+    }
+
     private List<String> getAllowedRedirectUris() {
         return settings("toolsets")
                 .getJsonObject("security", new JsonObject())

server/src/main/resources/aidial.settings.json

@@ -55,6 +55,10 @@
       "jclouds.filesystem.basedir": "data"
     }
   },
+  "storageLayout": {
+    "tenantRooted": false,
+    "defaultTenant": "default"
+  },
   "resources": {
     "maxSize" : 67108864,
     "maxSizeToCache": 1048576,

storage/src/main/java/com/epam/aidial/core/storage/resource/ResourceDescriptor.java

@@ -23,10 +23,6 @@ public class ResourceDescriptor {
     public static final String PLATFORM_BUCKET = "platform";
     public static final String PLATFORM_LOCATION = PLATFORM_BUCKET + PATH_SEPARATOR;
 
-    /**
-     * Layout in force for physical paths. Swapped once the tenant-rooted layout is enabled.
-     */
-    private static final StorageLayout LAYOUT = LegacyStorageLayout.INSTANCE;
 
     ResourceType type;
     /**
@@ -134,7 +130,8 @@ public String getAbsoluteFilePath() {
      * location followed by the resource-type folder.
      */
     private String getStoragePrefix() {
-        return LAYOUT.resolveLocationPrefix(bucketLocation) + LAYOUT.resolveTypeFolder(type.group()) + PATH_SEPARATOR;
+        StorageLayout layout = StorageLayouts.resolveActive();
+        return layout.resolveLocationPrefix(bucketLocation) + layout.resolveTypeFolder(type.group()) + PATH_SEPARATOR;
     }
 
     /**

storage/src/main/java/com/epam/aidial/core/storage/resource/StorageLayouts.java

@@ -0,0 +1,22 @@
+package com.epam.aidial.core.storage.resource;
+
+/**
+ * Holds the layout every physical path is composed with. Set once during start-up, before any resource
+ * is read or written; process-wide because {@link ResourceDescriptor} is constructed everywhere and
+ * carries no configuration of its own.
+ */
+public final class StorageLayouts {
+
+    private static volatile StorageLayout active = LegacyStorageLayout.INSTANCE;
+
+    private StorageLayouts() {
+    }
+
+    public static StorageLayout resolveActive() {
+        return active;
+    }
+
+    public static void useLayout(StorageLayout layout) {
+        active = layout;
+    }
+}

storage/src/main/java/com/epam/aidial/core/storage/resource/TenantRootedStorageLayout.java

@@ -0,0 +1,28 @@
+package com.epam.aidial.core.storage.resource;
+
+/**
+ * The tenant-rooted layout: every bucket location is placed under its tenant, and resource-type folders
+ * are reserved names. Conversion rules live in {@link TenantLayoutTransform}.
+ */
+public final class TenantRootedStorageLayout implements StorageLayout {
+
+    private final String tenantId;
+
+    public TenantRootedStorageLayout(String tenantId) {
+        if (tenantId == null || tenantId.isBlank()) {
+            throw new IllegalArgumentException("Tenant id must not be blank");
+        }
+
+        this.tenantId = tenantId;
+    }
+
+    @Override
+    public String resolveLocationPrefix(String bucketLocation) {
+        return TenantLayoutTransform.toTenantLocation(bucketLocation, tenantId);
+    }
+
+    @Override
+    public String resolveTypeFolder(String group) {
+        return TenantLayoutTransform.toTenantTypeFolder(group);
+    }
+}

storage/src/test/java/com/epam/aidial/core/storage/resource/StorageLayoutsTest.java

@@ -0,0 +1,42 @@
+package com.epam.aidial.core.storage.resource;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+
+public class StorageLayoutsTest {
+
+    @AfterEach
+    public void restoreDefaultLayout() {
+        StorageLayouts.useLayout(LegacyStorageLayout.INSTANCE);
+    }
+
+    @Test
+    public void testLegacyLayoutIsActiveByDefault() {
+        assertSame(LegacyStorageLayout.INSTANCE, StorageLayouts.resolveActive());
+    }
+
+    @Test
+    public void testActiveLayoutIsReplaceable() {
+        StorageLayout tenantRooted = new TenantRootedStorageLayout("acme");
+        StorageLayouts.useLayout(tenantRooted);
+
+        assertSame(tenantRooted, StorageLayouts.resolveActive());
+    }
+
+    @Test
+    public void testDescriptorPathFollowsActiveLayout() {
+        ResourceDescriptor file = new ResourceDescriptor(ResourceTypes.FILE, "notes.txt",
+                List.of("documents"), "bucket", "Users/u1/", false);
+
+        assertEquals("Users/u1/files/documents/notes.txt", file.getAbsoluteFilePath());
+
+        StorageLayouts.useLayout(new TenantRootedStorageLayout("acme"));
+
+        assertEquals(".org/acme/.users/u1/.files/documents/notes.txt", file.getAbsoluteFilePath());
+    }
+}

storage/src/test/java/com/epam/aidial/core/storage/resource/TenantRootedStorageLayoutTest.java

@@ -0,0 +1,36 @@
+package com.epam.aidial.core.storage.resource;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class TenantRootedStorageLayoutTest {
+
+    private final StorageLayout layout = new TenantRootedStorageLayout("acme");
+
+    @Test
+    public void testLocationPrefixIsTenantRooted() {
+        assertEquals(".org/acme/.users/u1/", layout.resolveLocationPrefix("Users/u1/"));
+        assertEquals(".org/acme/.keys/proj/", layout.resolveLocationPrefix("Keys/proj/"));
+        assertEquals(".org/acme/", layout.resolveLocationPrefix("public/"));
+        assertEquals("", layout.resolveLocationPrefix("platform/"));
+    }
+
+    @Test
+    public void testTypeFolderIsReserved() {
+        assertEquals(".files", layout.resolveTypeFolder("files"));
+        assertEquals(".conversations", layout.resolveTypeFolder("conversations"));
+    }
+
+    @Test
+    public void testUnsupportedLocationRejected() {
+        assertThrows(IllegalArgumentException.class, () -> layout.resolveLocationPrefix("Unknown/u1/"));
+    }
+
+    @Test
+    public void testBlankTenantRejected() {
+        assertThrows(IllegalArgumentException.class, () -> new TenantRootedStorageLayout(null));
+        assertThrows(IllegalArgumentException.class, () -> new TenantRootedStorageLayout(" "));
+    }
+}