test: verify a bucket migrated to the tenant-rooted layout #1870

server/src/test/java/com/epam/aidial/core/server/layout/BucketCopier.java

@@ -0,0 +1,115 @@
+package com.epam.aidial.core.server.layout;
+
+import com.epam.aidial.core.storage.resource.TenantLayoutTransformer;
+import lombok.SneakyThrows;
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.UserDefinedFileAttributeView;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+/**
+ * Copies a blob tree from the legacy layout into the tenant-rooted one.
+ *
+ * <p>A stand-in for the migrator P2 will build, not the migrator itself: it copies bytes and rewrites paths,
+ * with none of the throughput, resumability or re-encryption a real one needs. It exists so the verifier has
+ * something to verify before the real one lands, and so the questions the verifier asks are settled first.
+ */
+@Slf4j
+@UtilityClass
+public class BucketCopier {
+
+    /**
+     * Splits a physical path into the bucket location and what follows, by finding the resource-type folder.
+     * The two halves are what {@link TenantLayoutTransformer} converts, and a path is only made of those two
+     * plus the resource path within the type.
+     */
+    private record SplitPath(String location, String typeFolder, String rest) {
+    }
+
+    @SneakyThrows
+    public static int copy(Path legacyRoot, Path tenantRoot, String tenantId, List<String> typeFolders) {
+        int copied = 0;
+        for (Path source : list(legacyRoot)) {
+            String relative = legacyRoot.relativize(source).toString();
+            String transformed = toTenantPath(relative, tenantId, typeFolders);
+
+            Path destination = tenantRoot.resolve(transformed);
+            Files.createDirectories(destination.getParent());
+            Files.copy(source, destination);
+            copyMetadata(source, destination);
+            copied++;
+        }
+        return copied;
+    }
+
+    /**
+     * The path a migrated object lands at. Package-visible so the verifier maps source to destination the same
+     * way rather than reimplementing the rule.
+     */
+    static String toTenantPath(String legacyPath, String tenantId, List<String> typeFolders) {
+        SplitPath split = split(legacyPath, typeFolders);
+        return TenantLayoutTransformer.toTenantLocation(split.location(), tenantId)
+                + TenantLayoutTransformer.toTenantTypeFolder(split.typeFolder())
+                + "/" + split.rest();
+    }
+
+    private static SplitPath split(String legacyPath, List<String> typeFolders) {
+        String[] segments = legacyPath.split("/");
+        for (int i = 0; i < segments.length; i++) {
+            if (!typeFolders.contains(segments[i])) {
+                continue;
+            }
+            String location = i == 0 ? "" : String.join("/", List.of(segments).subList(0, i)) + "/";
+            String rest = String.join("/", List.of(segments).subList(i + 1, segments.length));
+            return new SplitPath(location, segments[i], rest);
+        }
+        throw new IllegalArgumentException("No resource type folder in: " + legacyPath);
+    }
+
+    /**
+     * Carries the blob's metadata across, not just its bytes.
+     *
+     * <p>The filesystem blob store keeps content-encoding, content-type and the user metadata — author,
+     * created_at, etag — in extended attributes, and a plain byte copy silently drops them. A compressed
+     * resource then arrives as gzip bytes nobody knows to decompress, and reads back as a parse error rather
+     * than as anything obviously missing.
+     */
+    @SneakyThrows
+    private static void copyMetadata(Path source, Path destination) {
+        UserDefinedFileAttributeView from = Files.getFileAttributeView(source, UserDefinedFileAttributeView.class);
+        UserDefinedFileAttributeView to = Files.getFileAttributeView(destination, UserDefinedFileAttributeView.class);
+        if (from == null || to == null) {
+            return;
+        }
+
+        for (String name : from.list()) {
+            ByteBuffer buffer = ByteBuffer.allocate(from.size(name));
+            from.read(name, buffer);
+            buffer.flip();
+            try {
+                to.write(name, buffer);
+            } catch (IOException systemAttribute) {
+                // Some attributes belong to the OS rather than the blob store and cannot be set by hand.
+                log.debug("Could not copy attribute {} of {}", name, source);
+            }
+        }
+    }
+
+    @SneakyThrows
+    private static List<Path> list(Path root) {
+        List<Path> files = new ArrayList<>();
+        try (Stream<Path> paths = Files.walk(root)) {
+            paths.filter(Files::isRegularFile).sorted().forEach(files::add);
+        } catch (IOException e) {
+            throw new IllegalStateException("Cannot read " + root, e);
+        }
+        return files;
+    }
+}

server/src/test/java/com/epam/aidial/core/server/layout/BucketVerifier.java

@@ -0,0 +1,163 @@
+package com.epam.aidial.core.server.layout;
+
+import lombok.SneakyThrows;
+import lombok.experimental.UtilityClass;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.UserDefinedFileAttributeView;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+/**
+ * Checks a bucket that has been copied to the tenant-rooted layout: everything arrived, and nothing extra did.
+ *
+ * <p>Object counts are not the check. A count that is off by one says something is wrong but not what, and a
+ * count that matches says nothing about <em>which</em> objects matched — so this compares the two trees as
+ * sets of paths mapped through the same transform the copier used, and names the one file whose absence would
+ * be silent.
+ */
+@UtilityClass
+public class BucketVerifier {
+
+    /**
+     * Governs read access to the whole public space. Its absence does not fail loudly: no document means an
+     * empty rule map, and {@code RuleMatcher} returns true for everything, so a migration that dropped it
+     * would publish the entire public space to everyone and every object count would still add up.
+     */
+    public static final String RULES_DOCUMENT = "public/rules/rules";
+
+    public record Result(int sourceObjects, int destinationObjects, List<String> problems) {
+        public boolean clean() {
+            return problems.isEmpty();
+        }
+    }
+
+    public static Result verify(Path legacyRoot, Path tenantRoot, String tenantId, List<String> typeFolders) {
+        Map<String, Path> source = index(legacyRoot);
+        Map<String, Path> destination = index(tenantRoot);
+
+        List<String> problems = new ArrayList<>();
+        Map<String, String> expected = new LinkedHashMap<>();
+        source.keySet().forEach(path -> expected.put(BucketCopier.toTenantPath(path, tenantId, typeFolders), path));
+
+        for (Map.Entry<String, String> entry : expected.entrySet()) {
+            Path arrived = destination.get(entry.getKey());
+            if (arrived == null) {
+                problems.add("missing at the destination: " + entry.getValue() + " → " + entry.getKey());
+                continue;
+            }
+            if (!sameContent(source.get(entry.getValue()), arrived)) {
+                problems.add("content differs: " + entry.getValue());
+            }
+            problems.addAll(compareMetadata(entry.getValue(), source.get(entry.getValue()), arrived));
+        }
+
+        destination.keySet().stream()
+                .filter(path -> !expected.containsKey(path))
+                .forEach(path -> problems.add("present at the destination with no source: " + path));
+
+        problems.addAll(verifyRulesDocument(source, destination, tenantId, typeFolders));
+        return new Result(source.size(), destination.size(), problems);
+    }
+
+    /**
+     * The rules document by name, not as one row in a count.
+     */
+    private static List<String> verifyRulesDocument(Map<String, Path> source, Map<String, Path> destination,
+                                                    String tenantId, List<String> typeFolders) {
+        String sourcePath = source.keySet().stream()
+                .filter(path -> path.endsWith(RULES_DOCUMENT))
+                .findFirst()
+                .orElse(null);
+        if (sourcePath == null) {
+            // Nothing was published, so there is no rules document to lose. Said out loud rather than passed
+            // over: a verifier that silently skips this check on an empty public space is one that would also
+            // skip it on a migration that dropped the file.
+            return List.of("note: the source has no " + RULES_DOCUMENT + ", so public access was not verified");
+        }
+
+        String expected = BucketCopier.toTenantPath(sourcePath, tenantId, typeFolders);
+        if (!destination.containsKey(expected)) {
+            return List.of("THE RULES DOCUMENT DID NOT ARRIVE: " + sourcePath + " → " + expected
+                    + ". Its absence fails open — the whole public space becomes readable by everyone.");
+        }
+        if (!sameContent(source.get(sourcePath), destination.get(expected))) {
+            return List.of("THE RULES DOCUMENT ARRIVED WITH DIFFERENT CONTENT: " + expected);
+        }
+        return List.of();
+    }
+
+    /**
+     * Compares the blob's metadata, not only its bytes.
+     *
+     * <p>Bytes alone are not enough to call an object migrated. Whether it is compressed is recorded beside
+     * it, not inside it, so a copy that keeps every byte and drops that flag reads back as a parse error —
+     * and an inventory that only checksums content reports such a bucket as perfectly migrated. The etag,
+     * author and creation time live in the same place.
+     */
+    @SneakyThrows
+    private static List<String> compareMetadata(String path, Path source, Path destination) {
+        Map<String, String> from = metadata(source);
+        Map<String, String> to = metadata(destination);
+        if (from.isEmpty() && to.isEmpty()) {
+            return List.of();
+        }
+
+        List<String> problems = new ArrayList<>();
+        for (Map.Entry<String, String> attribute : from.entrySet()) {
+            String arrived = to.get(attribute.getKey());
+            if (arrived == null) {
+                problems.add("metadata lost: " + path + " is missing '" + attribute.getKey()
+                        + "' at the destination (was: " + attribute.getValue() + ")");
+            } else if (!arrived.equals(attribute.getValue())) {
+                problems.add("metadata differs: " + path + " '" + attribute.getKey() + "' was "
+                        + attribute.getValue() + ", arrived as " + arrived);
+            }
+        }
+        return problems;
+    }
+
+    /**
+     * The blob store's own attributes. System attributes are skipped — they belong to the filesystem rather
+     * than to the object, and cannot be carried across by a copy.
+     */
+    @SneakyThrows
+    private static Map<String, String> metadata(Path file) {
+        UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
+        if (view == null) {
+            return Map.of();
+        }
+
+        Map<String, String> attributes = new LinkedHashMap<>();
+        for (String name : view.list()) {
+            if (name.startsWith("com.apple.")) {
+                continue;
+            }
+            ByteBuffer buffer = ByteBuffer.allocate(view.size(name));
+            view.read(name, buffer);
+            attributes.put(name, new String(buffer.array(), StandardCharsets.ISO_8859_1));
+        }
+        return attributes;
+    }
+
+    @SneakyThrows
+    private static boolean sameContent(Path left, Path right) {
+        return Files.mismatch(left, right) == -1;
+    }
+
+    @SneakyThrows
+    private static Map<String, Path> index(Path root) {
+        Map<String, Path> files = new LinkedHashMap<>();
+        try (Stream<Path> paths = Files.walk(root)) {
+            paths.filter(Files::isRegularFile).sorted()
+                    .forEach(path -> files.put(root.relativize(path).toString(), path));
+        }
+        return files;
+    }
+}

server/src/test/java/com/epam/aidial/core/server/layout/DialInstance.java

@@ -62,7 +62,9 @@ public class DialInstance implements AutoCloseable {
 
     private final AtomicLong nextId = new AtomicLong(FIRST_ID);
 
+    @Getter
     private final Path dataDir;
+    private final boolean keepData;
     private final RedisServer redis;
     private final AiDial dial;
     private final CloseableHttpClient client;
@@ -71,11 +73,22 @@ public class DialInstance implements AutoCloseable {
     private final String name;
     private final int port;
 
-    @SneakyThrows
     public DialInstance(String name, JsonObject layoutSettings, int redisPort) {
+        this(name, layoutSettings, redisPort, null);
+    }
+
+    /**
+     * @param existingData a blob tree to serve rather than start empty from — how the bucket verifier boots a
+     *                     core onto migrated data. It is left in place on close.
+     */
+    @SneakyThrows
+    public DialInstance(String name, JsonObject layoutSettings, int redisPort, Path existingData) {
         this.name = name;
-        this.dataDir = FileUtil.resolveRes("layout-diff-" + name);
-        FileUtil.deleteDir(dataDir);
+        this.keepData = existingData != null;
+        this.dataDir = existingData != null ? existingData : FileUtil.resolveRes("layout-diff-" + name);
+        if (existingData == null) {
+            FileUtil.deleteDir(dataDir);
+        }
         FileUtil.createDir(dataDir.resolve("test"));
 
         this.redis = RedisServer.newRedisServer()
@@ -281,7 +294,9 @@ private void closeQuietly() {
             } catch (Exception e) {
                 throw new IllegalStateException("Cannot stop redis for the " + name + " instance", e);
             }
-            FileUtil.deleteDir(dataDir);
+            if (!keepData) {
+                FileUtil.deleteDir(dataDir);
+            }
         }
     }
 }