fix: map system buckets and keep durable artifacts stable under the tenant-rooted layout #1863

README.md

@@ -232,8 +232,8 @@ without putting the raw address in the log.
 | storage.bucket              |     -      |    No    | Blob storage bucket.                                                                                                                                                                               |
 | storage.overrides.*         |     -      |    No    | Key-value pairs to override storage settings. `*` might be any specific blob storage setting to be overridden. Refer to [examples](#temporary-credentials-1) in the sections below.                |
 | storage.createBucket        |   false    |    No    | Indicates whether bucket should be created on start-up.                                                                                                                                            |
-| storageLayout.tenantRooted  |   false    |    No    | Places every bucket under a tenant root instead of storing it at the top level. Changing it on a populated deployment re-addresses existing data and requires migration; leave disabled otherwise.  |
-| storageLayout.defaultTenant |  default   |    No    | Tenant that existing buckets are placed under when `storageLayout.tenantRooted` is enabled.                                                                                                         |
+| storage.layout.tenantRooted  |   false    |    No    | Places every bucket under a tenant root instead of storing it at the top level. Changing it on a populated deployment re-addresses existing data and requires migration; leave disabled otherwise.  |
+| storage.layout.defaultTenant |  default   |    No    | Tenant that existing buckets are placed under when `storage.layout.tenantRooted` is enabled.                                                                                                         |
 | storage.prefix              |     -      |    No    | Base prefix for all stored resources. The purpose to use the same bucket for different environments, e.g. dev, prod, pre-prod. Must not contain path separators or any invalid chars.              |
 | storage.maxUploadedFileSize | 536870912  |    No    | Maximum size in bytes of uploaded file. If a size of uploaded file exceeds the limit the server returns HTTP code 413                                                                              |
 

server/src/main/java/com/epam/aidial/core/server/AiDial.java

@@ -210,10 +210,15 @@ void start() throws Exception {
                 accessTokenValidator = new AccessTokenValidator(settings("identityProviders"), vertx, taskExecutor, client, clientOptions, claimsLogLevel);
             }
 
-            StorageLayouts.useLayout(createStorageLayout(settings("storageLayout")));
+            StorageLayouts.useLayout(createStorageLayout(
+                    settings("storage").getJsonObject("layout", new JsonObject())));
 
             if (storage == null) {
-                Storage storageConfig = Json.decodeValue(settings("storage").toBuffer(), Storage.class);
+                // The layout block configures path composition, not the blob store; it is stripped
+                // before the decode because the codec rejects unknown properties.
+                JsonObject storageSettings = settings("storage").copy();
+                storageSettings.remove("layout");
+                Storage storageConfig = Json.decodeValue(storageSettings.toBuffer(), Storage.class);
                 storage = new BlobStorage(storageConfig);
             }
             encryptionService = new EncryptionService(settings("encryption"));

server/src/main/java/com/epam/aidial/core/server/config/SecretFieldProcessor.java

@@ -40,29 +40,30 @@ public void encryptFields(Object entity, ResourceDescriptor descriptor) {
         if (entity == null) {
             return;
         }
-        byte[] aad = descriptor.getAbsoluteFilePath().getBytes(StandardCharsets.UTF_8);
-        walk(entity, aad, true);
+        walk(entity, aad(descriptor), true);
     }
 
     public void decryptFields(Object entity, ResourceDescriptor descriptor) {
         if (entity == null) {
             return;
         }
-        byte[] aad = descriptor.getAbsoluteFilePath().getBytes(StandardCharsets.UTF_8);
-        walk(entity, aad, false);
+        walk(entity, aad(descriptor), false);
     }
 
     public String resolveSecret(String value, ResourceDescriptor descriptor) {
         if (value == null) {
             return null;
         }
         if (value.startsWith(ENC_PREFIX) && value.endsWith(ENC_SUFFIX)) {
-            byte[] aad = descriptor.getAbsoluteFilePath().getBytes(StandardCharsets.UTF_8);
-            return decryptEnvelope(value, aad, "value");
+            return decryptEnvelope(value, aad(descriptor), "value");
         }
         return value;
     }
 
+    private static byte[] aad(ResourceDescriptor descriptor) {
+        return descriptor.getLegacyFilePath().getBytes(StandardCharsets.UTF_8);
+    }
+
     /**
      * Strip every {@link EncryptedField}-annotated value (and any nested array elements that carry
      * the annotation) from {@code payload}. Used to project invalid-entity payloads on the admin

server/src/main/java/com/epam/aidial/core/server/security/ApiKeyStore.java

@@ -11,6 +11,7 @@
 import com.epam.aidial.core.storage.http.HttpStatus;
 import com.epam.aidial.core.storage.resource.ResourceDescriptor;
 import com.epam.aidial.core.storage.resource.ResourceTypes;
+import com.epam.aidial.core.storage.resource.SystemResourceRegistry;
 import com.epam.aidial.core.storage.util.RedisUtil;
 import io.vertx.core.Future;
 import io.vertx.core.json.JsonObject;
@@ -28,7 +29,6 @@
 import java.util.function.Function;
 
 import static com.epam.aidial.core.server.security.ApiKeyGenerator.generateKey;
-import static com.epam.aidial.core.storage.resource.ResourceDescriptor.PATH_SEPARATOR;
 
 /**
  * The store keeps per request and project API key data.
@@ -40,9 +40,6 @@
 @Slf4j
 public class ApiKeyStore {
 
-    public static final String API_KEY_DATA_BUCKET = "api_key_data";
-    public static final String API_KEY_DATA_LOCATION = API_KEY_DATA_BUCKET + PATH_SEPARATOR;
-
     private final AsyncTaskExecutor taskExecutor;
     private final RedissonClient redis;
     private final String prefix;
@@ -271,7 +268,7 @@ private void validateProjectKey(Key key) {
 
     private String toRedisKey(String apiKey) {
         ResourceDescriptor resource = ResourceDescriptorFactory.fromDecoded(
-                ResourceTypes.API_KEY_DATA, API_KEY_DATA_BUCKET, API_KEY_DATA_LOCATION, apiKey);
+                ResourceTypes.API_KEY_DATA, SystemResourceRegistry.API_KEY_DATA.bucket(), SystemResourceRegistry.API_KEY_DATA.location(), apiKey);
         return RedisUtil.redisKey(resource, prefix);
     }
 

server/src/main/java/com/epam/aidial/core/server/service/ApplicationService.java

@@ -814,14 +814,19 @@ private String deploymentLockKey(ResourceDescriptor resource) {
     }
 
     private String encodeTargetFolder(ResourceDescriptor resource, String id) {
-        String location = resource.getBucketLocation()
-                + DEPLOYMENTS_NAME + ResourceDescriptor.PATH_SEPARATOR
-                + id + ResourceDescriptor.PATH_SEPARATOR;
-
+        String location = deploymentFolderLocation(resource.getBucketLocation(), id);
         String name = encryptionService.encrypt(location);
         return ResourceDescriptorFactory.fromDecoded(ResourceTypes.FILE, name, location, null).getUrl();
     }
 
+    /**
+     * The synthesized sub-bucket location a function's deployment folder is keyed by. Package-visible so the
+     * layout test composes the same shape the service does rather than pinning a literal that can drift.
+     */
+    static String deploymentFolderLocation(String bucketLocation, String id) {
+        return bucketLocation + DEPLOYMENTS_NAME + ResourceDescriptor.PATH_SEPARATOR + id + ResourceDescriptor.PATH_SEPARATOR;
+    }
+
     public static boolean isActive(Application application) {
         return application != null && application.getFunction() != null && application.getFunction().getStatus().isActive();
     }

server/src/main/java/com/epam/aidial/core/server/service/BackgroundJobService.java

@@ -274,16 +274,18 @@ private Future<Void> processResult(
 
     private String encryptKey(ResourceDescriptor descriptor, String key) {
         BucketInfo bucketInfo = new BucketInfo(descriptor.getBucketName(), descriptor.getBucketLocation());
-        byte[] aad = descriptor.getAbsoluteFilePath().getBytes(StandardCharsets.UTF_8);
-        byte[] cipher = encryptionService.encrypt(bucketInfo, key.getBytes(StandardCharsets.UTF_8), aad);
+        byte[] cipher = encryptionService.encrypt(bucketInfo, key.getBytes(StandardCharsets.UTF_8), aad(descriptor));
         return Base64.getEncoder().encodeToString(cipher);
     }
 
     private String decryptKey(ResourceDescriptor descriptor, String key) {
         BucketInfo bucketInfo = new BucketInfo(descriptor.getBucketName(), descriptor.getBucketLocation());
-        byte[] aad = descriptor.getAbsoluteFilePath().getBytes(StandardCharsets.UTF_8);
         byte[] raw = Base64.getDecoder().decode(key);
-        return new String(encryptionService.decrypt(bucketInfo, raw, aad), StandardCharsets.UTF_8);
+        return new String(encryptionService.decrypt(bucketInfo, raw, aad(descriptor)), StandardCharsets.UTF_8);
+    }
+
+    private static byte[] aad(ResourceDescriptor descriptor) {
+        return descriptor.getLegacyFilePath().getBytes(StandardCharsets.UTF_8);
     }
 
     private Future<Void> completeAndProcess(

server/src/main/java/com/epam/aidial/core/server/service/InvitationService.java

@@ -269,7 +269,10 @@ public ResourceDescriptor getInvitationResource(String invitationId) {
         return ResourceDescriptorFactory.fromDecoded(resourceType, bucket, location, INVITATION_RESOURCE_FILENAME);
     }
 
+    // The id is handed out as a link and stored inside the invitations map, and getInvitationResource parses
+    // the location back out of it. It must therefore not carry the physical path, which the storage layout is
+    // free to change: an invitation issued before a layout change has to keep resolving after it.
     private String generateInvitationId(ResourceDescriptor resource) {
-        return encryptionService.encrypt(resource.getAbsoluteFilePath() + ResourceDescriptor.PATH_SEPARATOR + ApiKeyGenerator.generateKey());
+        return encryptionService.encrypt(resource.getLegacyFilePath() + ResourceDescriptor.PATH_SEPARATOR + ApiKeyGenerator.generateKey());
     }
 }

server/src/main/java/com/epam/aidial/core/server/service/ResponseMappingService.java

@@ -12,6 +12,7 @@
 import com.epam.aidial.core.storage.data.ResourceItemMetadata;
 import com.epam.aidial.core.storage.resource.ResourceDescriptor;
 import com.epam.aidial.core.storage.resource.ResourceTypes;
+import com.epam.aidial.core.storage.resource.SystemResourceRegistry;
 import com.epam.aidial.core.storage.service.ResourceService;
 import com.epam.aidial.core.storage.util.EtagHeader;
 import io.vertx.core.Vertx;
@@ -67,7 +68,7 @@ private Void cleanExpiredMappings() {
         log.debug("Housekeeping: scanning for expired response mappings");
         try {
             ResourceDescriptor root = ResourceDescriptorFactory.fromDecoded(
-                    ResourceTypes.RESPONSE_MAPPING, ResponseIdUtil.RESPONSE_MAPPINGS_BUCKET, ResponseIdUtil.RESPONSE_MAPPINGS_BUCKET_LOCATION, null);
+                    ResourceTypes.RESPONSE_MAPPING, SystemResourceRegistry.RESPONSE_MAPPINGS.bucket(), SystemResourceRegistry.RESPONSE_MAPPINGS.location(), null);
             cleanDeploymentSubfolders(root);
         } catch (Throwable e) {
             log.warn("Housekeeping: failed to clean expired response mappings", e);
@@ -96,7 +97,7 @@ private void cleanDeploymentSubfolders(ResourceDescriptor root) {
 
     private void cleanItemsInDeploymentFolder(String deploymentName) {
         ResourceDescriptor subfolder = ResourceDescriptorFactory.fromDecoded(
-                ResourceTypes.RESPONSE_MAPPING, ResponseIdUtil.RESPONSE_MAPPINGS_BUCKET, ResponseIdUtil.RESPONSE_MAPPINGS_BUCKET_LOCATION, deploymentName + "/");
+                ResourceTypes.RESPONSE_MAPPING, SystemResourceRegistry.RESPONSE_MAPPINGS.bucket(), SystemResourceRegistry.RESPONSE_MAPPINGS.location(), deploymentName + "/");
 
         long now = System.currentTimeMillis();
         String token = null;

server/src/main/java/com/epam/aidial/core/server/service/UserExternalServiceService.java

@@ -78,7 +78,7 @@ private static List<String> applicationSegments(String appPart) {
     public ExternalService put(String ownerUserId, String appPart, String serviceId, ExternalService service, String author) {
         ResourceDescriptor resource = descriptor(ownerUserId, appPart, serviceId);
         BucketInfo bucket = new BucketInfo(resource.getBucketName(), resource.getBucketLocation());
-        String aad = resource.getAbsoluteFilePath();
+        String aad = resource.getLegacyFilePath();
         MutableObject<ExternalService> result = new MutableObject<>();
         PersistedSecret persisted = new PersistedSecret();
         resourceService.computeResource(resource, EtagHeader.ANY, author, json -> {
@@ -104,7 +104,7 @@ public ExternalService get(String ownerUserId, String appPart, String serviceId)
             return null;
         }
         ExternalService service = ProxyUtil.convertToObject(stored.getValue(), ExternalService.class);
-        decryptSecret(resource.getAbsoluteFilePath(), new BucketInfo(resource.getBucketName(), resource.getBucketLocation()), service);
+        decryptSecret(resource.getLegacyFilePath(), new BucketInfo(resource.getBucketName(), resource.getBucketLocation()), service);
         return service;
     }
 

server/src/main/java/com/epam/aidial/core/server/token/TokenStatsTracker.java

@@ -6,6 +6,7 @@
 import com.epam.aidial.core.server.vertx.AsyncTaskExecutor;
 import com.epam.aidial.core.storage.resource.ResourceDescriptor;
 import com.epam.aidial.core.storage.resource.ResourceTypes;
+import com.epam.aidial.core.storage.resource.SystemResourceRegistry;
 import com.epam.aidial.core.storage.service.ResourceService;
 import com.epam.aidial.core.storage.util.EtagHeader;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@@ -21,14 +22,9 @@
 import java.util.List;
 import java.util.Map;
 
-import static com.epam.aidial.core.storage.resource.ResourceDescriptor.PATH_SEPARATOR;
-
 @Slf4j
 @RequiredArgsConstructor
 public class TokenStatsTracker {
-    public static final String DEPLOYMENT_COST_STATS_BUCKET = "deployment_cost_stats";
-    public static final String DEPLOYMENT_COST_STATS_LOCATION = DEPLOYMENT_COST_STATS_BUCKET + PATH_SEPARATOR;
-
     private final AsyncTaskExecutor taskExecutor;
     private final ResourceService resourceService;
 
@@ -216,6 +212,6 @@ public record UsageStats(TokenUsage total, List<UsagePerModel> usagePerModel) {
 
     private static ResourceDescriptor toResource(String traceId) {
         return ResourceDescriptorFactory.fromDecoded(
-                ResourceTypes.DEPLOYMENT_COST_STATS, DEPLOYMENT_COST_STATS_BUCKET, DEPLOYMENT_COST_STATS_LOCATION, traceId);
+                ResourceTypes.DEPLOYMENT_COST_STATS, SystemResourceRegistry.DEPLOYMENT_COST_STATS.bucket(), SystemResourceRegistry.DEPLOYMENT_COST_STATS.location(), traceId);
     }
 }

server/src/main/java/com/epam/aidial/core/server/util/BucketBuilder.java

@@ -3,6 +3,7 @@
 import com.epam.aidial.core.server.ProxyContext;
 import com.epam.aidial.core.server.data.AuthBucket;
 import com.epam.aidial.core.server.security.EncryptionService;
+import com.epam.aidial.core.storage.resource.ResourceDescriptor;
 import lombok.experimental.UtilityClass;
 
 import java.util.Objects;
@@ -12,8 +13,10 @@
 public class BucketBuilder {
 
     public static final String APPDATA_PATTERN = "appdata/%s";
-    public static final String USER_BUCKET_PATTERN = "Users/%s/";
-    public static final String API_KEY_BUCKET_PATTERN = "Keys/%s/";
+    // Prefixes declared in ResourceDescriptor: a storage layout recognizes principal locations by the
+    // same literals this builder formats them with.
+    public static final String USER_BUCKET_PATTERN = ResourceDescriptor.USERS_LOCATION_PREFIX + "%s" + ResourceDescriptor.PATH_SEPARATOR;
+    public static final String API_KEY_BUCKET_PATTERN = ResourceDescriptor.KEYS_LOCATION_PREFIX + "%s" + ResourceDescriptor.PATH_SEPARATOR;
 
     public String buildUserBucket(ProxyContext context) {
         if (context.getApiKeyData().getPerRequestKey() == null) {

server/src/main/java/com/epam/aidial/core/server/util/ResponseIdUtil.java

@@ -2,14 +2,11 @@
 
 import com.epam.aidial.core.storage.resource.ResourceDescriptor;
 import com.epam.aidial.core.storage.resource.ResourceTypes;
+import com.epam.aidial.core.storage.resource.SystemResourceRegistry;
 import lombok.experimental.UtilityClass;
 
 @UtilityClass
 public class ResponseIdUtil {
-    public static final String RESPONSE_MAPPINGS_BUCKET = "response_mappings";
-    public static final String RESPONSE_MAPPINGS_BUCKET_LOCATION = RESPONSE_MAPPINGS_BUCKET + "/";
-    public static final String BACKGROUND_JOB_BUCKET = "background_jobs";
-    public static final String BACKGROUND_JOB_BUCKET_LOCATION = BACKGROUND_JOB_BUCKET + "/";
     public static final String RESPONSE_ID_PREFIX = "dial_";
 
     public String createResponseId(String deploymentName, String uuid) {
@@ -28,11 +25,11 @@ public ResourceDescriptor getResponseMappingDescriptor(String dialResponseId) {
         String uuid = dialResponseId.substring(underscore + 1);
         String relativePath = deploymentName + "/" + uuid;
         return ResourceDescriptorFactory.fromDecoded(
-                ResourceTypes.RESPONSE_MAPPING, RESPONSE_MAPPINGS_BUCKET, RESPONSE_MAPPINGS_BUCKET_LOCATION, relativePath);
+                ResourceTypes.RESPONSE_MAPPING, SystemResourceRegistry.RESPONSE_MAPPINGS.bucket(), SystemResourceRegistry.RESPONSE_MAPPINGS.location(), relativePath);
     }
 
     public ResourceDescriptor getBackgroundJobDescriptor(String jobId) {
         return ResourceDescriptorFactory.fromDecoded(
-                ResourceTypes.BACKGROUND_JOB, BACKGROUND_JOB_BUCKET, BACKGROUND_JOB_BUCKET_LOCATION, jobId);
+                ResourceTypes.BACKGROUND_JOB, SystemResourceRegistry.BACKGROUND_JOBS.bucket(), SystemResourceRegistry.BACKGROUND_JOBS.location(), jobId);
     }
 }

server/src/main/resources/aidial.settings.json

@@ -53,12 +53,12 @@
     "createBucket": true,
     "overrides": {
       "jclouds.filesystem.basedir": "data"
+    },
+    "layout": {
+      "tenantRooted": false,
+      "defaultTenant": "default"
     }
   },
-  "storageLayout": {
-    "tenantRooted": false,
-    "defaultTenant": "default"
-  },
   "resources": {
     "maxSize" : 67108864,
     "maxSizeToCache": 1048576,

server/src/test/java/com/epam/aidial/core/server/TenantRootedLayoutApiTest.java

@@ -14,10 +14,11 @@
 import java.util.stream.Stream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
- * Drives the resource API with {@code storageLayout.tenantRooted} enabled: the whole stack — descriptor,
+ * Drives the resource API with {@code storage.layout.tenantRooted} enabled: the whole stack — descriptor,
  * cache and blob store — has to agree on the tenant-rooted paths, which unit tests cannot show.
  */
 public class TenantRootedLayoutApiTest extends ResourceBaseTest {
@@ -26,9 +27,9 @@ public class TenantRootedLayoutApiTest extends ResourceBaseTest {
 
     @Override
     protected JsonObject additionalSettingsOverrides() {
-        return new JsonObject().put("storageLayout", new JsonObject()
+        return new JsonObject().put("storage", new JsonObject().put("layout", new JsonObject()
                 .put("tenantRooted", true)
-                .put("defaultTenant", TENANT));
+                .put("defaultTenant", TENANT)));
     }
 
     @AfterEach
@@ -66,21 +67,19 @@ public void testBlobIsStoredUnderTenantRoot() throws IOException {
         Response flushed = resourceRequest(HttpMethod.GET, "/folder/conversation");
         assertEquals(200, flushed.status());
 
-        List<Path> storedPaths = findStoredPaths("");
+        List<Path> storedPaths = findStoredPaths();
         List<Path> tenantRootedPaths = storedPaths.stream()
                 .filter(path -> path.toString().contains(".org/" + TENANT))
                 .toList();
-        assertTrue(!tenantRootedPaths.isEmpty(),
+        assertFalse(tenantRootedPaths.isEmpty(),
                 () -> "No blob stored under the tenant root, found: " + storedPaths);
         assertTrue(tenantRootedPaths.stream().anyMatch(path -> path.toString().contains(".conversations")),
                 () -> "Conversations are not stored in a reserved type folder: " + tenantRootedPaths);
     }
 
-    private List<Path> findStoredPaths(String marker) throws IOException {
+    private List<Path> findStoredPaths() throws IOException {
         try (Stream<Path> paths = Files.walk(testDir)) {
-            return paths.filter(Files::isRegularFile)
-                    .filter(path -> path.toString().contains(marker))
-                    .toList();
+            return paths.filter(Files::isRegularFile).toList();
         }
     }
 }