diff --git a/server/src/test/java/com/epam/aidial/core/server/layout/BucketCopier.java b/server/src/test/java/com/epam/aidial/core/server/layout/BucketCopier.java new file mode 100644 index 000000000..9026089a9 --- /dev/null +++ b/server/src/test/java/com/epam/aidial/core/server/layout/BucketCopier.java @@ -0,0 +1,115 @@ +package com.epam.aidial.core.server.layout; + +import com.epam.aidial.core.storage.resource.TenantLayoutTransformer; +import lombok.SneakyThrows; +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.UserDefinedFileAttributeView; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +/** + * Copies a blob tree from the legacy layout into the tenant-rooted one. + * + *

A stand-in for the migrator P2 will build, not the migrator itself: it copies bytes and rewrites paths, + * with none of the throughput, resumability or re-encryption a real one needs. It exists so the verifier has + * something to verify before the real one lands, and so the questions the verifier asks are settled first. + */ +@Slf4j +@UtilityClass +public class BucketCopier { + + /** + * Splits a physical path into the bucket location and what follows, by finding the resource-type folder. + * The two halves are what {@link TenantLayoutTransformer} converts, and a path is only made of those two + * plus the resource path within the type. + */ + private record SplitPath(String location, String typeFolder, String rest) { + } + + @SneakyThrows + public static int copy(Path legacyRoot, Path tenantRoot, String tenantId, List typeFolders) { + int copied = 0; + for (Path source : list(legacyRoot)) { + String relative = legacyRoot.relativize(source).toString(); + String transformed = toTenantPath(relative, tenantId, typeFolders); + + Path destination = tenantRoot.resolve(transformed); + Files.createDirectories(destination.getParent()); + Files.copy(source, destination); + copyMetadata(source, destination); + copied++; + } + return copied; + } + + /** + * The path a migrated object lands at. Package-visible so the verifier maps source to destination the same + * way rather than reimplementing the rule. + */ + static String toTenantPath(String legacyPath, String tenantId, List typeFolders) { + SplitPath split = split(legacyPath, typeFolders); + return TenantLayoutTransformer.toTenantLocation(split.location(), tenantId) + + TenantLayoutTransformer.toTenantTypeFolder(split.typeFolder()) + + "/" + split.rest(); + } + + private static SplitPath split(String legacyPath, List typeFolders) { + String[] segments = legacyPath.split("/"); + for (int i = 0; i < segments.length; i++) { + if (!typeFolders.contains(segments[i])) { + continue; + } + String location = i == 0 ? "" : String.join("/", List.of(segments).subList(0, i)) + "/"; + String rest = String.join("/", List.of(segments).subList(i + 1, segments.length)); + return new SplitPath(location, segments[i], rest); + } + throw new IllegalArgumentException("No resource type folder in: " + legacyPath); + } + + /** + * Carries the blob's metadata across, not just its bytes. + * + *

The filesystem blob store keeps content-encoding, content-type and the user metadata — author, + * created_at, etag — in extended attributes, and a plain byte copy silently drops them. A compressed + * resource then arrives as gzip bytes nobody knows to decompress, and reads back as a parse error rather + * than as anything obviously missing. + */ + @SneakyThrows + private static void copyMetadata(Path source, Path destination) { + UserDefinedFileAttributeView from = Files.getFileAttributeView(source, UserDefinedFileAttributeView.class); + UserDefinedFileAttributeView to = Files.getFileAttributeView(destination, UserDefinedFileAttributeView.class); + if (from == null || to == null) { + return; + } + + for (String name : from.list()) { + ByteBuffer buffer = ByteBuffer.allocate(from.size(name)); + from.read(name, buffer); + buffer.flip(); + try { + to.write(name, buffer); + } catch (IOException systemAttribute) { + // Some attributes belong to the OS rather than the blob store and cannot be set by hand. + log.debug("Could not copy attribute {} of {}", name, source); + } + } + } + + @SneakyThrows + private static List list(Path root) { + List files = new ArrayList<>(); + try (Stream paths = Files.walk(root)) { + paths.filter(Files::isRegularFile).sorted().forEach(files::add); + } catch (IOException e) { + throw new IllegalStateException("Cannot read " + root, e); + } + return files; + } +} diff --git a/server/src/test/java/com/epam/aidial/core/server/layout/BucketVerifier.java b/server/src/test/java/com/epam/aidial/core/server/layout/BucketVerifier.java new file mode 100644 index 000000000..2873ef908 --- /dev/null +++ b/server/src/test/java/com/epam/aidial/core/server/layout/BucketVerifier.java @@ -0,0 +1,163 @@ +package com.epam.aidial.core.server.layout; + +import lombok.SneakyThrows; +import lombok.experimental.UtilityClass; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.UserDefinedFileAttributeView; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +/** + * Checks a bucket that has been copied to the tenant-rooted layout: everything arrived, and nothing extra did. + * + *

Object counts are not the check. A count that is off by one says something is wrong but not what, and a + * count that matches says nothing about which objects matched — so this compares the two trees as + * sets of paths mapped through the same transform the copier used, and names the one file whose absence would + * be silent. + */ +@UtilityClass +public class BucketVerifier { + + /** + * Governs read access to the whole public space. Its absence does not fail loudly: no document means an + * empty rule map, and {@code RuleMatcher} returns true for everything, so a migration that dropped it + * would publish the entire public space to everyone and every object count would still add up. + */ + public static final String RULES_DOCUMENT = "public/rules/rules"; + + public record Result(int sourceObjects, int destinationObjects, List problems) { + public boolean clean() { + return problems.isEmpty(); + } + } + + public static Result verify(Path legacyRoot, Path tenantRoot, String tenantId, List typeFolders) { + Map source = index(legacyRoot); + Map destination = index(tenantRoot); + + List problems = new ArrayList<>(); + Map expected = new LinkedHashMap<>(); + source.keySet().forEach(path -> expected.put(BucketCopier.toTenantPath(path, tenantId, typeFolders), path)); + + for (Map.Entry entry : expected.entrySet()) { + Path arrived = destination.get(entry.getKey()); + if (arrived == null) { + problems.add("missing at the destination: " + entry.getValue() + " → " + entry.getKey()); + continue; + } + if (!sameContent(source.get(entry.getValue()), arrived)) { + problems.add("content differs: " + entry.getValue()); + } + problems.addAll(compareMetadata(entry.getValue(), source.get(entry.getValue()), arrived)); + } + + destination.keySet().stream() + .filter(path -> !expected.containsKey(path)) + .forEach(path -> problems.add("present at the destination with no source: " + path)); + + problems.addAll(verifyRulesDocument(source, destination, tenantId, typeFolders)); + return new Result(source.size(), destination.size(), problems); + } + + /** + * The rules document by name, not as one row in a count. + */ + private static List verifyRulesDocument(Map source, Map destination, + String tenantId, List typeFolders) { + String sourcePath = source.keySet().stream() + .filter(path -> path.endsWith(RULES_DOCUMENT)) + .findFirst() + .orElse(null); + if (sourcePath == null) { + // Nothing was published, so there is no rules document to lose. Said out loud rather than passed + // over: a verifier that silently skips this check on an empty public space is one that would also + // skip it on a migration that dropped the file. + return List.of("note: the source has no " + RULES_DOCUMENT + ", so public access was not verified"); + } + + String expected = BucketCopier.toTenantPath(sourcePath, tenantId, typeFolders); + if (!destination.containsKey(expected)) { + return List.of("THE RULES DOCUMENT DID NOT ARRIVE: " + sourcePath + " → " + expected + + ". Its absence fails open — the whole public space becomes readable by everyone."); + } + if (!sameContent(source.get(sourcePath), destination.get(expected))) { + return List.of("THE RULES DOCUMENT ARRIVED WITH DIFFERENT CONTENT: " + expected); + } + return List.of(); + } + + /** + * Compares the blob's metadata, not only its bytes. + * + *

Bytes alone are not enough to call an object migrated. Whether it is compressed is recorded beside + * it, not inside it, so a copy that keeps every byte and drops that flag reads back as a parse error — + * and an inventory that only checksums content reports such a bucket as perfectly migrated. The etag, + * author and creation time live in the same place. + */ + @SneakyThrows + private static List compareMetadata(String path, Path source, Path destination) { + Map from = metadata(source); + Map to = metadata(destination); + if (from.isEmpty() && to.isEmpty()) { + return List.of(); + } + + List problems = new ArrayList<>(); + for (Map.Entry attribute : from.entrySet()) { + String arrived = to.get(attribute.getKey()); + if (arrived == null) { + problems.add("metadata lost: " + path + " is missing '" + attribute.getKey() + + "' at the destination (was: " + attribute.getValue() + ")"); + } else if (!arrived.equals(attribute.getValue())) { + problems.add("metadata differs: " + path + " '" + attribute.getKey() + "' was " + + attribute.getValue() + ", arrived as " + arrived); + } + } + return problems; + } + + /** + * The blob store's own attributes. System attributes are skipped — they belong to the filesystem rather + * than to the object, and cannot be carried across by a copy. + */ + @SneakyThrows + private static Map metadata(Path file) { + UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class); + if (view == null) { + return Map.of(); + } + + Map attributes = new LinkedHashMap<>(); + for (String name : view.list()) { + if (name.startsWith("com.apple.")) { + continue; + } + ByteBuffer buffer = ByteBuffer.allocate(view.size(name)); + view.read(name, buffer); + attributes.put(name, new String(buffer.array(), StandardCharsets.ISO_8859_1)); + } + return attributes; + } + + @SneakyThrows + private static boolean sameContent(Path left, Path right) { + return Files.mismatch(left, right) == -1; + } + + @SneakyThrows + private static Map index(Path root) { + Map files = new LinkedHashMap<>(); + try (Stream paths = Files.walk(root)) { + paths.filter(Files::isRegularFile).sorted() + .forEach(path -> files.put(root.relativize(path).toString(), path)); + } + return files; + } +} diff --git a/server/src/test/java/com/epam/aidial/core/server/layout/DialInstance.java b/server/src/test/java/com/epam/aidial/core/server/layout/DialInstance.java index ea88855bb..1201c9069 100644 --- a/server/src/test/java/com/epam/aidial/core/server/layout/DialInstance.java +++ b/server/src/test/java/com/epam/aidial/core/server/layout/DialInstance.java @@ -62,7 +62,9 @@ public class DialInstance implements AutoCloseable { private final AtomicLong nextId = new AtomicLong(FIRST_ID); + @Getter private final Path dataDir; + private final boolean keepData; private final RedisServer redis; private final AiDial dial; private final CloseableHttpClient client; @@ -71,11 +73,22 @@ public class DialInstance implements AutoCloseable { private final String name; private final int port; - @SneakyThrows public DialInstance(String name, JsonObject layoutSettings, int redisPort) { + this(name, layoutSettings, redisPort, null); + } + + /** + * @param existingData a blob tree to serve rather than start empty from — how the bucket verifier boots a + * core onto migrated data. It is left in place on close. + */ + @SneakyThrows + public DialInstance(String name, JsonObject layoutSettings, int redisPort, Path existingData) { this.name = name; - this.dataDir = FileUtil.resolveRes("layout-diff-" + name); - FileUtil.deleteDir(dataDir); + this.keepData = existingData != null; + this.dataDir = existingData != null ? existingData : FileUtil.resolveRes("layout-diff-" + name); + if (existingData == null) { + FileUtil.deleteDir(dataDir); + } FileUtil.createDir(dataDir.resolve("test")); this.redis = RedisServer.newRedisServer() @@ -281,7 +294,9 @@ private void closeQuietly() { } catch (Exception e) { throw new IllegalStateException("Cannot stop redis for the " + name + " instance", e); } - FileUtil.deleteDir(dataDir); + if (!keepData) { + FileUtil.deleteDir(dataDir); + } } } } diff --git a/server/src/test/java/com/epam/aidial/core/server/layout/LayoutBucketVerifierTest.java b/server/src/test/java/com/epam/aidial/core/server/layout/LayoutBucketVerifierTest.java new file mode 100644 index 000000000..398be6201 --- /dev/null +++ b/server/src/test/java/com/epam/aidial/core/server/layout/LayoutBucketVerifierTest.java @@ -0,0 +1,197 @@ +package com.epam.aidial.core.server.layout; + +import com.epam.aidial.core.server.FileUtil; +import com.epam.aidial.core.storage.resource.LegacyStorageLayout; +import com.epam.aidial.core.storage.resource.ResourceTypes; +import com.epam.aidial.core.storage.resource.StorageLayouts; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.json.JsonObject; +import lombok.SneakyThrows; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Migrates a bucket and then serves it: copy a legacy tree into the tenant-rooted layout, check everything + * arrived, and boot a tenant-rooted core onto the copy to read it all back. + * + *

This is the one instrument that crosses layouts. The other two start empty and each run reads only its + * own writes, so neither can see a failure that only happens to data written under one layout and read under + * the other — which is the entire risk P2 carries. + */ +@Tag("layout-diff") +public class LayoutBucketVerifierTest { + + private static final String TENANT = "migrated-tenant"; + + private static final int SOURCE_REDIS_PORT = 16375; + private static final int MIGRATED_REDIS_PORT = 16376; + + /** + * Where the blob store puts objects inside the data directory: the configured bucket, then the prefix. + */ + private static final String BLOB_ROOT = "test/test-2"; + + private static final List TYPE_FOLDERS = + Arrays.stream(ResourceTypes.values()).map(ResourceTypes::group).distinct().toList(); + + @AfterEach + public void restoreLegacyLayout() { + StorageLayouts.useLayout(LegacyStorageLayout.INSTANCE); + } + + @Test + @SneakyThrows + public void testMigratedBucketIsCompleteAndReadable() { + List seed = CorpusRunner.loadCorpus(CorpusRunner.ACCESS_CORPUS); + + Path legacyData = FileUtil.resolveRes("layout-diff-migration-source"); + Path migratedData = FileUtil.resolveRes("layout-diff-migration-destination"); + FileUtil.deleteDir(legacyData); + FileUtil.deleteDir(migratedData); + + Seeded seeded = seedOnLegacyLayout(legacyData, seed); + Map variables = seeded.variables(); + + Path legacyRoot = legacyData.resolve(BLOB_ROOT); + Path migratedRoot = migratedData.resolve(BLOB_ROOT); + int copied = BucketCopier.copy(legacyRoot, migratedRoot, TENANT, TYPE_FOLDERS); + assertTrue(copied > 0, "nothing was copied, so nothing below verifies anything"); + + BucketVerifier.Result result = BucketVerifier.verify(legacyRoot, migratedRoot, TENANT, TYPE_FOLDERS); + if (!result.clean()) { + fail("Migrated bucket did not verify (" + result.sourceObjects() + " source objects, " + + result.destinationObjects() + " destination):\n " + String.join("\n ", result.problems())); + } + + readBackOnTenantRootedLayout(migratedData, seeded); + } + + /** + * Writes a representative bucket on the legacy layout and leaves it on disk. + */ + private record Seeded(Map variables, Map readBack) { + } + + private static Seeded seedOnLegacyLayout(Path dataDir, List seed) throws Exception { + try (DialInstance instance = + new DialInstance("migration-source", new JsonObject().put("tenantRooted", false), + SOURCE_REDIS_PORT, dataDir)) { + CorpusRunner.Run run = CorpusRunner.replay(instance, seed); + // What the resources read back as before anything moved. The question is whether migration + // changes that, not whether every read is a 200 — some fixtures are legitimately not readable, + // and asserting 200 outright would make the suite fail for reasons that have nothing to do with + // the layout. + Map readBack = readAll(instance, run.variables()); + // Writes land in Redis first and are flushed behind the request; copying before that would copy a + // tree that is missing whatever had not been written out yet. + Thread.sleep(5000); + return new Seeded(run.variables(), readBack); + } finally { + StorageLayouts.useLayout(LegacyStorageLayout.INSTANCE); + } + } + + /** + * The external-service entry authenticates as the user who owns it: user-authored services live in the + * caller's own user bucket, which an api key does not have. + */ + private record ReadBack(String url, Map headers) { + static ReadBack byApiKey(String url) { + return new ReadBack(url, Map.of("api-key", "proxyKey1")); + } + } + + private static final String EXTERNAL_SERVICE_URL = "applications/${svcOwnerBucket}/svcapp/external-services/billing"; + + private static final List READ_BACK = List.of( + ReadBack.byApiKey("conversations/${bucket1}/access/own"), + ReadBack.byApiKey("conversations/${bucket1}/access/shared"), + ReadBack.byApiKey("conversations/${bucket1}/access/writable"), + ReadBack.byApiKey("conversations/${bucket1}/access/publishable"), + ReadBack.byApiKey("applications/${bucket1}/access/app"), + ReadBack.byApiKey("files/${bucket1}/appdata/testapp/data.txt"), + ReadBack.byApiKey("conversations/public/access/published"), + new ReadBack(EXTERNAL_SERVICE_URL, Map.of("authorization", "svc-owner"))); + + private static Map readAll(DialInstance instance, Map variables) { + Map results = new LinkedHashMap<>(); + for (ReadBack readBack : READ_BACK) { + RecordedResponse response = instance.send(HttpMethod.GET.name(), + "/v1/" + resolve(readBack.url(), variables), null, null, readBack.headers(), null); + results.put(readBack.url(), response.status() + " " + response.body()); + } + return results; + } + + /** + * Boots a tenant-rooted core onto the migrated tree, with an empty cache, and reads the seeded resources + * back through the API. Inventory says the bytes arrived; only this says they can still be served. + */ + private static void readBackOnTenantRootedLayout(Path migratedData, Seeded seeded) { + Map variables = seeded.variables(); + try (DialInstance instance = new DialInstance("migration-destination", new JsonObject() + .put("tenantRooted", true) + .put("defaultTenant", TENANT), MIGRATED_REDIS_PORT, migratedData)) { + + assertEquals(variables.get("bucket1"), instance.bucket("proxyKey1"), + "the migrated core resolves a different bucket, so nothing below addresses the same data"); + + Map after = readAll(instance, variables); + assertTrue(after.values().stream().anyMatch(result -> result.startsWith("200")), + () -> "nothing at all was readable after migration, so this proves nothing: " + after); + + List changed = READ_BACK.stream() + .map(ReadBack::url) + .filter(url -> !seeded.readBack().get(url).equals(after.get(url))) + .map(url -> " " + url + "\n before migration: " + seeded.readBack().get(url) + + "\n after migration: " + after.get(url)) + .toList(); + if (!changed.isEmpty()) { + fail("Migration changed what these resources read back as:\n" + String.join("\n", changed)); + } + + // The share and the publication have to survive too: they are the state that makes the data + // usable rather than merely present. + RecordedResponse shared = instance.send("POST", "/v1/ops/resource/share/list", null, + "{\"resourceTypes\":[\"CONVERSATION\"],\"with\":\"me\"}", + Map.of("api-key", "proxyKey2"), null); + assertEquals(200, shared.status(), () -> "share listing failed after migration: " + shared.body()); + assertTrue(shared.body().contains("access/shared"), + () -> "the share did not survive migration: " + shared.body()); + + RecordedResponse publicRead = instance.send(HttpMethod.GET.name(), + "/v1/conversations/public/access/published", null, null, + Map.of("api-key", "proxyKey2"), null); + assertEquals(200, publicRead.status(), + () -> "published resource is not readable after migration: " + publicRead.body()); + + // Named check, like the rules document: this resource stores an AAD-encrypted client secret, and + // serving it requires the migrated ciphertext to decrypt. Before/after equality alone would also + // pass if both reads failed identically, which for the one encrypted fixture is not good enough. + assertTrue(after.get(EXTERNAL_SERVICE_URL).startsWith("200"), + () -> "the encrypted external service does not decrypt after migration: " + + after.get(EXTERNAL_SERVICE_URL)); + } finally { + StorageLayouts.useLayout(LegacyStorageLayout.INSTANCE); + } + } + + private static String resolve(String template, Map variables) { + String resolved = template; + for (Map.Entry variable : variables.entrySet()) { + resolved = resolved.replace("${" + variable.getKey() + "}", variable.getValue()); + } + return resolved; + } +} diff --git a/server/src/test/resources/layout-diff/access-corpus/01-seed.json b/server/src/test/resources/layout-diff/access-corpus/01-seed.json index 90d44577f..2bd3093ef 100644 --- a/server/src/test/resources/layout-diff/access-corpus/01-seed.json +++ b/server/src/test/resources/layout-diff/access-corpus/01-seed.json @@ -268,6 +268,50 @@ "assistantModelId": "a", "lastActivityDate": 6 } + }, + { + "name": "svc-owner-bucket", + "method": "GET", + "path": "/v1/bucket", + "headers": { + "authorization": "svc-owner" + }, + "capture": { + "svcOwnerBucket": { + "at": "/bucket" + } + } + }, + { + "name": "create-svc-app", + "method": "PUT", + "path": "/v1/applications/${svcOwnerBucket}/svcapp", + "headers": { + "authorization": "svc-owner" + }, + "body": { + "display_name": "svc app", + "endpoint": "http://localhost:9999/completions", + "description": "carries a user-authored external service with an encrypted client secret" + } + }, + { + "name": "create-external-service", + "method": "PUT", + "path": "/v1/applications/${svcOwnerBucket}/svcapp/external-services/billing", + "headers": { + "authorization": "svc-owner" + }, + "body": { + "display_name": "Billing", + "auth_settings": { + "authentication_type": "OAUTH", + "client_id": "layout-diff-client", + "client_secret": "layout-diff-secret", + "authorization_endpoint": "http://localhost:9999/authorize", + "token_endpoint": "http://localhost:9999/token" + } + } } ] }