diff --git a/pom.xml b/pom.xml index aa06d2a88b..92d1f3fbb2 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ 2.11.12 2.11 2.28.2 - 1.20.4 + 1.21.4 1.68 3.7.1 2.5.3 @@ -594,6 +594,12 @@ ${testcontainers.version} test + + org.testcontainers + testcontainers + ${testcontainers.version} + test + org.testcontainers postgresql diff --git a/smart-ozone-support/pom.xml b/smart-ozone-support/pom.xml index f66cff76dd..217a49773d 100644 --- a/smart-ozone-support/pom.xml +++ b/smart-ozone-support/pom.xml @@ -35,6 +35,7 @@ smart-ozone smart-ozone-fs-client + smart-ozone-common \ No newline at end of file diff --git a/smart-ozone-support/smart-ozone-common/pom.xml b/smart-ozone-support/smart-ozone-common/pom.xml new file mode 100644 index 0000000000..cfdbaba89d --- /dev/null +++ b/smart-ozone-support/smart-ozone-common/pom.xml @@ -0,0 +1,61 @@ + + + + 4.0.0 + + + org.smartdata + smart-ozone-support + 2.2.0-SNAPSHOT + + + smart-ozone-common + 2.2.0-SNAPSHOT + jar + + + + org.smartdata + smart-hadoop-common + 2.2.0-SNAPSHOT + + + + org.smartdata + smart-common + 2.2.0-SNAPSHOT + + + + org.apache.ozone + ozone-filesystem-hadoop3 + ${ozone.version} + + + + org.testcontainers + testcontainers + test + + + \ No newline at end of file diff --git a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneSmartConf.java b/smart-ozone-support/smart-ozone-common/src/main/java/org/smartdata/ozone/OzoneSmartConf.java similarity index 100% rename from smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/OzoneSmartConf.java rename to smart-ozone-support/smart-ozone-common/src/main/java/org/smartdata/ozone/OzoneSmartConf.java diff --git a/smart-ozone-support/smart-ozone-common/src/test/java/org/smartdata/ozone/OzoneClusterCompose.java b/smart-ozone-support/smart-ozone-common/src/test/java/org/smartdata/ozone/OzoneClusterCompose.java new file mode 100644 index 0000000000..eece5c7227 --- /dev/null +++ b/smart-ozone-support/smart-ozone-common/src/test/java/org/smartdata/ozone/OzoneClusterCompose.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.ozone; + +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.io.File; + +import static org.smartdata.ozone.OzoneClusterHarness.resourceAbsolutePath; + +public class OzoneClusterCompose extends ComposeContainer { + public static final String DEFAULT_COMPOSE_FILE = "docker-compose-ozone.yaml"; + + private static final String OM_SERVICE = "om"; + private static final String SCM_SERVICE = "scm"; + private static final String DATANODE_SERVICE = "datanode"; + + private static final int OM_PORT = 9862; + private static final int SCM_PORT = 9876; + private static final int DATANODE_PORT = 9864; + + public OzoneClusterCompose() { + this(resourceAbsolutePath(DEFAULT_COMPOSE_FILE)); + } + + public OzoneClusterCompose(String composeFilePath) { + super(new File(composeFilePath)); + + withExposedService(DATANODE_SERVICE, DATANODE_PORT, + Wait.forLogMessage(".*Ozone container server started.*", 1)); + withExposedService(SCM_SERVICE, SCM_PORT, + Wait.forLogMessage(".*SCM exiting safe mode.*", 1)); + withExposedService(OM_SERVICE, OM_PORT, + Wait.forLogMessage(".*Leader om1@.* is ready.*", 1)); + } + + public String getOmHost() { + return getServiceHost(OM_SERVICE, OM_PORT); + } + + public int getOmPort() { + return getServicePort(OM_SERVICE, OM_PORT); + } + + public String getOmRpcAddress() { + return getOmHost() + ":" + getOmPort(); + } + +} diff --git a/smart-ozone-support/smart-ozone-common/src/test/java/org/smartdata/ozone/OzoneClusterHarness.java b/smart-ozone-support/smart-ozone-common/src/test/java/org/smartdata/ozone/OzoneClusterHarness.java new file mode 100644 index 0000000000..6f108cac09 --- /dev/null +++ b/smart-ozone-support/smart-ozone-common/src/test/java/org/smartdata/ozone/OzoneClusterHarness.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.ozone; + +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.smartdata.conf.SmartConf; + +import java.io.IOException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Optional; + +import static org.smartdata.ozone.OzoneSmartConf.DEFAULT_OFS_ADDRESS; + +public class OzoneClusterHarness { + + private static final ReplicationConfig REPLICATION_CONFIG = + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE); + + @Rule + public OzoneClusterCompose ozoneContainer = new OzoneClusterCompose(); + + protected OzoneSmartConf ozoneConf; + protected ObjectStore ozoneClient; + + @Before + public void init() throws Exception { + SmartConf smartConf = new SmartConf(); + smartConf.set(DEFAULT_OFS_ADDRESS, "ofs://" + ozoneContainer.getOmRpcAddress()); + smartConf.set("ozone.om.address", ozoneContainer.getOmRpcAddress()); + + ozoneConf = new OzoneSmartConf(smartConf); + ozoneClient = OzoneClientFactory.getRpcClient(ozoneConf).getObjectStore(); + } + + @After + public void tearDown() throws IOException { + ozoneClient.getClientProxy().close(); + } + + public static void createKey(OzoneBucket bucket, String key, String data) throws IOException { + byte[] dataBytes = data.getBytes(StandardCharsets.UTF_8); + try (OzoneOutputStream outputStream = + bucket.createKey(key, dataBytes.length, REPLICATION_CONFIG, new HashMap<>())) { + outputStream.write(dataBytes); + outputStream.flush(); + } + } + + public static String resourceAbsolutePath(String relativePath) { + return Optional.ofNullable( + OzoneClusterHarness.class.getClassLoader().getResource(relativePath)) + .map(URL::getPath) + .orElseThrow(() -> new RuntimeException("Resource not found")); + } +} diff --git a/smart-ozone-support/smart-ozone-common/src/test/resources/docker-compose-ozone.yaml b/smart-ozone-support/smart-ozone-common/src/test/resources/docker-compose-ozone.yaml new file mode 100644 index 0000000000..29e2a8e35c --- /dev/null +++ b/smart-ozone-support/smart-ozone-common/src/test/resources/docker-compose-ozone.yaml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +x-image: + &image + image: ${OZONE_IMAGE:-apache/ozone}:${OZONE_IMAGE_VERSION:-2.0.0}${OZONE_IMAGE_FLAVOR:-} + +x-common-config: + &common-config + OZONE-SITE.XML_hdds.datanode.dir: "/data/hdds" + OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata" + OZONE-SITE.XML_ozone.om.address: "om" + OZONE-SITE.XML_ozone.administrators: "hadoop,om,hdfs,${ADMIN_USER:-root}" + OZONE-SITE.XML_ozone.om.http-address: "om:9874" + OZONE-SITE.XML_ozone.replication: "1" + OZONE-SITE.XML_ozone.scm.block.client.address: "scm" + OZONE-SITE.XML_ozone.scm.client.address: "scm" + OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data/metadata" + OZONE-SITE.XML_ozone.scm.names: "scm" + no_proxy: "om,scm,localhost,127.0.0.1" + +version: "3" +services: + datanode: + <<: *image + ports: + - 9864 + command: [ "ozone","datanode" ] + environment: + <<: *common-config + + om: + <<: *image + ports: + - 9862 + environment: + <<: *common-config + CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts: "*" + CORE-SITE.XML_hadoop.proxyuser.hadoop.groups: "*" + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + WAITFOR: scm:9876 + command: [ "ozone","om" ] + + scm: + <<: *image + ports: + - 9876 + environment: + <<: *common-config + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: [ "ozone","scm" ] diff --git a/smart-ozone-support/smart-ozone-fs-client/pom.xml b/smart-ozone-support/smart-ozone-fs-client/pom.xml index 136e8ef103..8ba6a076cf 100644 --- a/smart-ozone-support/smart-ozone-fs-client/pom.xml +++ b/smart-ozone-support/smart-ozone-fs-client/pom.xml @@ -35,7 +35,7 @@ org.smartdata - smart-hadoop-common + smart-ozone-common 2.2.0-SNAPSHOT @@ -43,16 +43,6 @@ smart-client 2.2.0-SNAPSHOT - - org.smartdata - smart-common - 2.2.0-SNAPSHOT - - - org.apache.ozone - ozone-filesystem-hadoop3 - ${ozone.version} - org.projectlombok lombok @@ -78,6 +68,18 @@ mockito-core test + + org.smartdata + smart-ozone-common + 2.2.0-SNAPSHOT + test + test-jar + + + org.testcontainers + testcontainers + test + diff --git a/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/FileAccessReportSupport.java b/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/FileAccessReportSupport.java index 37726f4fef..45112b46a3 100644 --- a/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/FileAccessReportSupport.java +++ b/smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/FileAccessReportSupport.java @@ -44,7 +44,7 @@ public FileAccessReportSupport(SmartClientProtocol ssmClient, String... basePath } public void reportFileAccess(String path) { - String pathWithoutAuthority = new Path(basePath, path) + String pathWithoutAuthority = new Path(basePath, removeLeadingSlash(path)) .toUri() .getPath(); @@ -60,6 +60,10 @@ public void reportFileAccess(String path) { } } + private String removeLeadingSlash(String path) { + return path.startsWith("/") ? path.substring(1) : path; + } + @Override public void close() throws IOException { ssmClient.close(); diff --git a/smart-ozone-support/smart-ozone-fs-client/src/test/java/org/apache/hadoop/fs/ozone/SmartOzoneClientAdapterTest.java b/smart-ozone-support/smart-ozone-fs-client/src/test/java/org/apache/hadoop/fs/ozone/SmartOzoneClientAdapterTest.java index 11824a701f..a67732f920 100644 --- a/smart-ozone-support/smart-ozone-fs-client/src/test/java/org/apache/hadoop/fs/ozone/SmartOzoneClientAdapterTest.java +++ b/smart-ozone-support/smart-ozone-fs-client/src/test/java/org/apache/hadoop/fs/ozone/SmartOzoneClientAdapterTest.java @@ -18,17 +18,20 @@ package org.apache.hadoop.fs.ozone; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.smartdata.metrics.FileAccessEvent; import org.smartdata.model.FileState; +import org.smartdata.ozone.OzoneClusterHarness; import org.smartdata.ozone.client.SmartOzoneClientAdapter; import org.smartdata.protocol.SmartClientProtocol; import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -36,56 +39,81 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -public class SmartOzoneClientAdapterTest { +public class SmartOzoneClientAdapterTest extends OzoneClusterHarness { + private static final String TEST_VOLUME = "vol1"; + private static final String TEST_BUCKET = "buck1"; + + private static final String TEST_DATA = "data_777"; private MockSsmClient ssmClient; @Before - public void init() { + public void initClient() throws IOException { this.ssmClient = new MockSsmClient(); + + ozoneClient.createVolume(TEST_VOLUME); + ozoneClient.getVolume(TEST_VOLUME) + .createBucket(TEST_BUCKET); } @Test - @Ignore("Unignore when testing environment for Ozone will be added (ADH-7291)") - public void testReportAccessEventOfs() throws IOException { + public void testReportAccessEventOfs() throws Exception { SmartOzoneClientAdapter clientAdapter = new SmartRootedOzoneFileSystem.SmartClientAdapter( - "TODO", -1, new OzoneConfiguration(), null, ssmClient); - testReportAccessEventInternal(clientAdapter); + ozoneContainer.getOmHost(), ozoneContainer.getOmPort(), + ozoneConf, null, ssmClient); + testReportAccessEventInternal(clientAdapter, TEST_VOLUME, TEST_BUCKET); } @Test - @Ignore("Unignore when testing environment for Ozone will be added (ADH-7291)") - public void testReportAccessEventO3fs() throws IOException { + public void testReportAccessEventO3fs() throws Exception { SmartOzoneClientAdapter clientAdapter = new SmartOzoneFileSystem.SmartClientAdapter( - "TODO", -1, new OzoneConfiguration(), "someVolume", "someBucket", null, ssmClient); + ozoneContainer.getOmHost(), ozoneContainer.getOmPort(), + ozoneConf, TEST_VOLUME, TEST_BUCKET, null, ssmClient); testReportAccessEventInternal(clientAdapter); } - public void testReportAccessEventInternal(SmartOzoneClientAdapter clientAdapter) throws IOException { - clientAdapter.createFile("key", (short) 1, false, false); + private void testReportAccessEventInternal(SmartOzoneClientAdapter clientAdapter, String... prefixSegments) + throws Exception { + String keyPrefix = Arrays.stream(prefixSegments) + .collect(Collectors.joining("/", "", "/")); + + createFile(clientAdapter, keyPrefix + "key"); + createFile(clientAdapter, keyPrefix + "anotherKey"); + createFile(clientAdapter, keyPrefix + "keyToRemove"); assertTrue(ssmClient.accessEvents.isEmpty()); - clientAdapter.getFileStatus("key1", null, null, null); + clientAdapter.getFileStatus(keyPrefix + "anotherKey", + new URI("ofs://test:7070"), new Path(keyPrefix, "anotherKey"), "anon"); assertTrue(ssmClient.accessEvents.isEmpty()); - clientAdapter.readFile("someKey"); + clientAdapter.deleteObject(keyPrefix + "keyToRemove"); + assertTrue(ssmClient.accessEvents.isEmpty()); + + clientAdapter.readFile(keyPrefix + "key").close(); assertEquals(1, ssmClient.accessEvents.size()); - clientAdapter.readFile("someDir/anotherKey"); + clientAdapter.readFile(keyPrefix + "anotherKey").close(); assertEquals(2, ssmClient.accessEvents.size()); List actualAccessedFiles = ssmClient.accessEvents.stream() .map(FileAccessEvent::getPath) .collect(Collectors.toList()); - List expectedAccessFiles = Stream.of("someKey", "someDir/anotherKey") - .map(path -> new Path(clientAdapter.getBasePath(), path)) + List expectedAccessFiles = Stream.of("key", "anotherKey") + .map(path -> new Path(new Path("/" + TEST_VOLUME, TEST_BUCKET), path)) .map(path -> path.toUri().getPath()) .collect(Collectors.toList()); assertEquals(expectedAccessFiles, actualAccessedFiles); } + private void createFile(SmartOzoneClientAdapter clientAdapter, String key) throws IOException { + try (OzoneFSOutputStream outputStream = clientAdapter.createFile(key, (short) 1, true, true)) { + outputStream.write(TEST_DATA.getBytes(StandardCharsets.UTF_8)); + outputStream.flush(); + } + } + private static class MockSsmClient implements SmartClientProtocol { private final List accessEvents = new ArrayList<>(); diff --git a/smart-ozone-support/smart-ozone-fs-client/src/test/resources/log4j2.properties b/smart-ozone-support/smart-ozone-fs-client/src/test/resources/log4j2.properties new file mode 100644 index 0000000000..9eaa484e2a --- /dev/null +++ b/smart-ozone-support/smart-ozone-fs-client/src/test/resources/log4j2.properties @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# log4j configuration used during build and unit tests + +rootLogger = INFO, STDOUT + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n diff --git a/smart-ozone-support/smart-ozone/pom.xml b/smart-ozone-support/smart-ozone/pom.xml index add152833b..b1e6e86388 100644 --- a/smart-ozone-support/smart-ozone/pom.xml +++ b/smart-ozone-support/smart-ozone/pom.xml @@ -36,19 +36,9 @@ org.smartdata - smart-hadoop-common + smart-ozone-common 2.2.0-SNAPSHOT - - org.smartdata - smart-common - 2.2.0-SNAPSHOT - - - org.apache.ozone - ozone-filesystem-hadoop3 - ${ozone.version} - org.projectlombok lombok @@ -84,5 +74,17 @@ mockito-core test + + org.smartdata + smart-ozone-common + 2.2.0-SNAPSHOT + test + test-jar + + + org.testcontainers + testcontainers + test + \ No newline at end of file diff --git a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/snapshot/OfsSnapshotFetcher.java b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/snapshot/OfsSnapshotFetcher.java index 341b68a2fe..42f12930df 100644 --- a/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/snapshot/OfsSnapshotFetcher.java +++ b/smart-ozone-support/smart-ozone/src/main/java/org/smartdata/ozone/snapshot/OfsSnapshotFetcher.java @@ -148,7 +148,7 @@ private CompletableFuture handleVolume(OzoneVolume volume) { private CompletableFuture handleVolume(FileStatus fileStatus) { OzoneFileInfo.Builder fileBuilder = OzoneFileInfo.builder() - .path(fileStatus.getPath().toString()) + .path(pathWithoutAuthority(fileStatus.getPath())) .isVolume(true); return saveFile(fileStatus, fileBuilder) .thenComposeAsync(ignore -> executeInParallel( @@ -158,7 +158,7 @@ private CompletableFuture handleVolume(FileStatus fileStatus) { private CompletableFuture handleBucket(FileStatus fileStatus) { OzoneFileInfo.Builder fileBuilder = OzoneFileInfo.builder() - .path(fileStatus.getPath().toString()) + .path(pathWithoutAuthority(fileStatus.getPath())) .isBucket(true); return saveFile(fileStatus, fileBuilder) .thenComposeAsync(ignore -> createBucketSnapshot(fileStatus), executor) @@ -172,7 +172,7 @@ private CompletableFuture handleBucket(FileStatus fileStatus) { private CompletableFuture handleKey(FileStatus fileStatus) { Path filePath = getOriginalFilePath(fileStatus.getPath()); OzoneFileInfo.Builder fileBuilder = OzoneFileInfo.builder() - .path(filePath.toString()); + .path(pathWithoutAuthority(filePath)); return saveFile(fileStatus, fileBuilder) .thenComposeAsync(ignore -> handleChildrenIfDirectory(fileStatus), executor); } @@ -360,6 +360,10 @@ Path getOriginalFilePath(Path snapshotPath) { ); } + private String pathWithoutAuthority(Path path) { + return path.toUri().getPath(); + } + private String ofsSnapshotKeyToPath(OFSPath ofsPath) { String ofsKey = ofsPath.getKeyName(); int snapshotsDirEnd = ofsKey.indexOf(PATH_DELIMITER); diff --git a/smart-ozone-support/smart-ozone/src/test/java/org/smartdata/ozone/snapshot/OfsSnapshotFetcherTest.java b/smart-ozone-support/smart-ozone/src/test/java/org/smartdata/ozone/snapshot/OfsSnapshotFetcherTest.java index 487f50b0e6..9eeb6197cf 100644 --- a/smart-ozone-support/smart-ozone/src/test/java/org/smartdata/ozone/snapshot/OfsSnapshotFetcherTest.java +++ b/smart-ozone-support/smart-ozone/src/test/java/org/smartdata/ozone/snapshot/OfsSnapshotFetcherTest.java @@ -17,40 +17,235 @@ */ package org.smartdata.ozone.snapshot; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import lombok.Data; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.junit.After; +import org.junit.Before; import org.junit.Test; +import org.smartdata.ozone.OzoneClusterHarness; +import org.smartdata.ozone.model.FsObjectStreamRecord; +import org.smartdata.ozone.model.OzoneFileInfo; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.util.stream.IntStream.range; import static org.junit.Assert.assertEquals; -public class OfsSnapshotFetcherTest { +public class OfsSnapshotFetcherTest extends OzoneClusterHarness { + private static final int SNAPSHOT_FETCH_TIMEOUT_SEC = 10; + + private static final int VOLUMES_COUNT = 2; + private static final int BUCKETS_PER_VOL_COUNT = 4; + private static final int KEYS_PER_BUCKET_COUNT = 3; + private static final int DIRS_PER_BUCKET_COUNT = 4; + private static final int SUBDIRS_PER_DIR_COUNT = 3; + private static final int KEYS_PER_DIR_COUNT = 3; + private static final int KEYS_PER_SUBDIR_COUNT = 2; + + private final static int TOTAL_EVENTS_PER_DIR_COUNT = + 1 + KEYS_PER_DIR_COUNT + SUBDIRS_PER_DIR_COUNT * (1 + KEYS_PER_SUBDIR_COUNT); + + private final static int TOTAL_EVENTS_PER_BUCKET_COUNT = + 1 + KEYS_PER_BUCKET_COUNT + DIRS_PER_BUCKET_COUNT * TOTAL_EVENTS_PER_DIR_COUNT; + + private final static int TOTAL_EVENTS_PER_VOLUME_COUNT = + 1 + BUCKETS_PER_VOL_COUNT * TOTAL_EVENTS_PER_BUCKET_COUNT; + + // 1 for EOF event and 1 for s3v bucket for s3 related keys + private final static int TOTAL_EVENTS_COUNT = + 2 + VOLUMES_COUNT * TOTAL_EVENTS_PER_VOLUME_COUNT; + + private OfsSnapshotFetcher fetcher; + + @Before + public void initFetcher() throws IOException { + fetcher = OfsSnapshotFetcher.builder() + .fs(FileSystem.get(ozoneConf)) + .objectStore(ozoneClient) + .conf(ozoneConf) + .executor(Executors.newFixedThreadPool(8)) + .batchSize(TOTAL_EVENTS_COUNT + 2) + .build(); + + initFs(); + } + + @After + public void closeFetcher() { + fetcher.close(); + } @Test - public void testExtractOriginalFilePath() { - testExtractOriginalFilePath("/vol/buck/.snapshot/sn1/key", "/vol/buck/key"); - testExtractOriginalFilePath("/vol/buck/.snapshot/sn1/dir/key", "/vol/buck/dir/key"); - testExtractOriginalFilePath("/vol/buck/.snapshot/sn1/dir/subdir/key", - "/vol/buck/dir/subdir/key"); + public void testFetchKeys() throws Exception { + fetcher.runSnapshotAsync().get(SNAPSHOT_FETCH_TIMEOUT_SEC, TimeUnit.SECONDS); + + BlockingQueue outputQueue = fetcher.getOutputQueue(); + assertEquals(TOTAL_EVENTS_COUNT, outputQueue.size()); + + Set actualFiles = outputQueue.stream() + .filter(OzoneFileInfo.class::isInstance) + .map(OzoneFileInfo.class::cast) + .map(this::toFileView) + .collect(Collectors.toSet()); + + assertEquals(expectedFiles(), actualFiles); + } + + private FileView toFileView(OzoneFileInfo fileInfo) { + return new FileView( + fileInfo.getPath(), + fileInfo.getLength(), + fileInfo.isVolume(), + fileInfo.isBucket() + ); + } + + private void initFs() { + range(0, VOLUMES_COUNT) + .forEach(this::initVolume); + } + + private Set expectedFiles() { + Set expected = new HashSet<>(); + + expected.add(volumeInfo("/s3v")); + + range(0, VOLUMES_COUNT).forEach(vIdx -> { + String volPath = "/vol" + vIdx; + expected.add(volumeInfo(volPath)); + + range(0, BUCKETS_PER_VOL_COUNT).forEach(bIdx -> { + String buckPath = volPath + "/buck" + bIdx; + expected.add(bucketInfo(buckPath)); + + range(0, KEYS_PER_BUCKET_COUNT).forEach(kIdx -> + expected.add(keyInfo(buckPath + "/key" + kIdx))); + + range(0, DIRS_PER_BUCKET_COUNT).forEach(dIdx -> { + String dirPath = buckPath + "/dir" + dIdx; + expected.add(dirInfo(dirPath)); + + range(0, KEYS_PER_DIR_COUNT).forEach(kIdx -> + expected.add(keyInfo(dirPath + "/key" + kIdx))); + + range(0, SUBDIRS_PER_DIR_COUNT).forEach(sdIdx -> { + String sdPath = dirPath + "/subdir" + sdIdx; + expected.add(dirInfo(sdPath)); + + range(0, KEYS_PER_SUBDIR_COUNT).forEach(kIdx -> + expected.add(keyInfo(sdPath + "/key" + kIdx))); + }); + }); + }); + }); - testExtractOriginalFilePath("ofs://host:123/vol/buck/.snapshot/sn1/dir/key", - "ofs://host:123/vol/buck/dir/key"); + return expected; + } + + private FileView volumeInfo(String path) { + return new FileView(path, 0, true, false); + } + + private FileView bucketInfo(String path) { + return new FileView(path, 0, false, true); + } + + private FileView dirInfo(String path) { + return new FileView(path, 0, false, false); + } + + private FileView keyInfo(String path) { + String fileData = Arrays.stream(path.split("/")) + // skip root, volume and bucket + .skip(3) + .collect(Collectors.joining("/")) + "_data"; - testExtractOriginalFilePath("/", "/"); - testExtractOriginalFilePath("/vol", "/vol"); - testExtractOriginalFilePath("/vol/buck", "/vol/buck"); + return new FileView(path, + fileData.getBytes(StandardCharsets.UTF_8).length, false, false); } - private void testExtractOriginalFilePath(String sourcePath, String expectedPath) { - try (OfsSnapshotFetcher fetcher = emptyFetcher()) { - Path actualPath = fetcher.getOriginalFilePath(new Path(sourcePath)); - assertEquals(expectedPath, actualPath.toString()); + private void initVolume(int volumeIdx) { + try { + String volumeName = "vol" + volumeIdx; + ozoneClient.createVolume(volumeName); + OzoneVolume volume = ozoneClient.getVolume(volumeName); + + range(0, BUCKETS_PER_VOL_COUNT) + .forEach(idx -> initBucket(volume, idx)); + } catch (IOException e) { + throw new RuntimeException("Failed to initialize volume " + volumeIdx, e); } } - private OfsSnapshotFetcher emptyFetcher() { - return OfsSnapshotFetcher.builder() - .conf(new OzoneConfiguration()) - .batchSize(1) - .build(); + private void initBucket(OzoneVolume volume, int bucketIdx) { + try { + String bucketName = "buck" + bucketIdx; + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + range(0, KEYS_PER_BUCKET_COUNT).forEach(idx -> + createKey(bucket, "key" + idx)); + + range(0, DIRS_PER_BUCKET_COUNT).forEach(idx -> + initDirectory(bucket, idx)); + + } catch (IOException e) { + throw new RuntimeException("Failed to initialize bucket " + bucketIdx, e); + } + } + + private void initDirectory(OzoneBucket bucket, int dirIdx) { + try { + String dirPath = "dir" + dirIdx; + bucket.createDirectory(dirPath); + + range(0, KEYS_PER_DIR_COUNT).forEach(idx -> + createKey(bucket, dirPath + "/key" + idx)); + + range(0, SUBDIRS_PER_DIR_COUNT).forEach(idx -> + initSubdirectory(bucket, dirPath, idx)); + + } catch (IOException e) { + throw new RuntimeException("Failed to initialize directory " + dirIdx, e); + } + } + + private void initSubdirectory(OzoneBucket bucket, String parentPath, int dirIdx) { + try { + String subDirPath = parentPath + "/subdir" + dirIdx; + bucket.createDirectory(subDirPath); + + range(0, KEYS_PER_SUBDIR_COUNT).forEach(idx -> + createKey(bucket, subDirPath + "/key" + idx)); + + } catch (IOException e) { + throw new RuntimeException("Failed to initialize subdirectory " + dirIdx, e); + } + } + + private void createKey(OzoneBucket bucket, String keyName) { + try { + createKey(bucket, keyName, keyName + "_data"); + } catch (IOException e) { + throw new RuntimeException("Failed to create key: " + keyName, e); + } + } + + @Data + private static class FileView { + private final String path; + private final long length; + private final boolean isVolume; + private final boolean isBucket; } } \ No newline at end of file diff --git a/smart-ozone-support/smart-ozone/src/test/java/org/smartdata/ozone/snapshot/OfsSnapshotPathTransformerTest.java b/smart-ozone-support/smart-ozone/src/test/java/org/smartdata/ozone/snapshot/OfsSnapshotPathTransformerTest.java new file mode 100644 index 0000000000..c7c63360da --- /dev/null +++ b/smart-ozone-support/smart-ozone/src/test/java/org/smartdata/ozone/snapshot/OfsSnapshotPathTransformerTest.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.ozone.snapshot; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class OfsSnapshotPathTransformerTest { + + @Test + public void testExtractOriginalFilePath() { + testExtractOriginalFilePath("/vol/buck/.snapshot/sn1/key", "/vol/buck/key"); + testExtractOriginalFilePath("/vol/buck/.snapshot/sn1/dir/key", "/vol/buck/dir/key"); + testExtractOriginalFilePath("/vol/buck/.snapshot/sn1/dir/subdir/key", + "/vol/buck/dir/subdir/key"); + + testExtractOriginalFilePath("ofs://host:123/vol/buck/.snapshot/sn1/dir/key", + "ofs://host:123/vol/buck/dir/key"); + + testExtractOriginalFilePath("/", "/"); + testExtractOriginalFilePath("/vol", "/vol"); + testExtractOriginalFilePath("/vol/buck", "/vol/buck"); + } + + private void testExtractOriginalFilePath(String sourcePath, String expectedPath) { + try (OfsSnapshotFetcher fetcher = emptyFetcher()) { + Path actualPath = fetcher.getOriginalFilePath(new Path(sourcePath)); + assertEquals(expectedPath, actualPath.toString()); + } + } + + private OfsSnapshotFetcher emptyFetcher() { + return OfsSnapshotFetcher.builder() + .conf(new OzoneConfiguration()) + .batchSize(1) + .build(); + } +} \ No newline at end of file diff --git a/smart-ozone-support/smart-ozone/src/test/resources/log4j2.properties b/smart-ozone-support/smart-ozone/src/test/resources/log4j2.properties new file mode 100644 index 0000000000..9eaa484e2a --- /dev/null +++ b/smart-ozone-support/smart-ozone/src/test/resources/log4j2.properties @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# log4j configuration used during build and unit tests + +rootLogger = INFO, STDOUT + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n diff --git a/smart-server/pom.xml b/smart-server/pom.xml index 1eff0e8404..c1f26dcde9 100644 --- a/smart-server/pom.xml +++ b/smart-server/pom.xml @@ -198,6 +198,13 @@ test test-jar + + org.smartdata + smart-ozone-common + 2.2.0-SNAPSHOT + test + test-jar + org.testcontainers jdbc diff --git a/smart-server/src/test/java/org/smartdata/server/OzoneSmartClusterHarness.java b/smart-server/src/test/java/org/smartdata/server/OzoneSmartClusterHarness.java new file mode 100644 index 0000000000..dc381c29d9 --- /dev/null +++ b/smart-server/src/test/java/org/smartdata/server/OzoneSmartClusterHarness.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.server; + +import org.smartdata.ozone.OzoneClusterHarness; + +// TODO init SmartServer when ADH-7056 will be completed +public class OzoneSmartClusterHarness extends OzoneClusterHarness { + protected SmartServer ssm; +}