[ADH-7291] Add Docker test environment for Ozone

pom.xml

@@ -71,7 +71,7 @@
     <scala.version>2.11.12</scala.version>
     <scala.binary.version>2.11</scala.binary.version>
     <mockito.version>2.28.2</mockito.version>
-    <testcontainers.version>1.20.4</testcontainers.version>
+    <testcontainers.version>1.21.4</testcontainers.version>
     <bouncycastle.version>1.68</bouncycastle.version>
     <protobuf.version>3.7.1</protobuf.version>
     <dbunitVersion>2.5.3</dbunitVersion>
@@ -594,6 +594,12 @@
         <version>${testcontainers.version}</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>org.testcontainers</groupId>
+        <artifactId>testcontainers</artifactId>
+        <version>${testcontainers.version}</version>
+        <scope>test</scope>
+      </dependency>
       <dependency>
         <groupId>org.testcontainers</groupId>
         <artifactId>postgresql</artifactId>

smart-ozone-support/pom.xml

@@ -35,6 +35,7 @@
     <modules>
         <module>smart-ozone</module>
         <module>smart-ozone-fs-client</module>
+        <module>smart-ozone-common</module>
     </modules>
 
 </project>

smart-ozone-support/smart-ozone-common/pom.xml

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.smartdata</groupId>
+        <artifactId>smart-ozone-support</artifactId>
+        <version>2.2.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>smart-ozone-common</artifactId>
+    <version>2.2.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.smartdata</groupId>
+            <artifactId>smart-hadoop-common</artifactId>
+            <version>2.2.0-SNAPSHOT</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.smartdata</groupId>
+            <artifactId>smart-common</artifactId>
+            <version>2.2.0-SNAPSHOT</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.ozone</groupId>
+            <artifactId>ozone-filesystem-hadoop3</artifactId>
+            <version>${ozone.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+</project>

smart-ozone-support/smart-ozone-common/src/test/java/org/smartdata/ozone/OzoneClusterCompose.java

@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.smartdata.ozone;
+
+import org.testcontainers.containers.ComposeContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+
+import java.io.File;
+
+import static org.smartdata.ozone.OzoneClusterHarness.resourceAbsolutePath;
+
+public class OzoneClusterCompose extends ComposeContainer {
+  public static final String DEFAULT_COMPOSE_FILE = "docker-compose-ozone.yaml";
+
+  private static final String OM_SERVICE = "om";
+  private static final String SCM_SERVICE = "scm";
+  private static final String DATANODE_SERVICE = "datanode";
+
+  private static final int OM_PORT = 9862;
+  private static final int SCM_PORT = 9876;
+  private static final int DATANODE_PORT = 9864;
+
+  public OzoneClusterCompose() {
+    this(resourceAbsolutePath(DEFAULT_COMPOSE_FILE));
+  }
+
+  public OzoneClusterCompose(String composeFilePath) {
+    super(new File(composeFilePath));
+
+    withExposedService(DATANODE_SERVICE, DATANODE_PORT,
+        Wait.forLogMessage(".*Ozone container server started.*", 1));
+    withExposedService(SCM_SERVICE, SCM_PORT,
+        Wait.forLogMessage(".*SCM exiting safe mode.*", 1));
+    withExposedService(OM_SERVICE, OM_PORT,
+        Wait.forLogMessage(".*Leader om1@.* is ready.*", 1));
+  }
+
+  public String getOmHost() {
+    return getServiceHost(OM_SERVICE, OM_PORT);
+  }
+
+  public int getOmPort() {
+    return getServicePort(OM_SERVICE, OM_PORT);
+  }
+
+  public String getOmRpcAddress() {
+    return getOmHost() + ":" + getOmPort();
+  }
+
+}

smart-ozone-support/smart-ozone-common/src/test/java/org/smartdata/ozone/OzoneClusterHarness.java

@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.smartdata.ozone;
+
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.smartdata.conf.SmartConf;
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Optional;
+
+import static org.smartdata.ozone.OzoneSmartConf.DEFAULT_OFS_ADDRESS;
+
+public class OzoneClusterHarness {
+
+  private static final ReplicationConfig REPLICATION_CONFIG =
+      ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE);
+
+  @Rule
+  public OzoneClusterCompose ozoneContainer = new OzoneClusterCompose();
+
+  protected OzoneSmartConf ozoneConf;
+  protected ObjectStore ozoneClient;
+
+  @Before
+  public void init() throws Exception {
+    SmartConf smartConf = new SmartConf();
+    smartConf.set(DEFAULT_OFS_ADDRESS, "ofs://" + ozoneContainer.getOmRpcAddress());
+    smartConf.set("ozone.om.address", ozoneContainer.getOmRpcAddress());
+
+    ozoneConf = new OzoneSmartConf(smartConf);
+    ozoneClient = OzoneClientFactory.getRpcClient(ozoneConf).getObjectStore();
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    ozoneClient.getClientProxy().close();
+  }
+
+  public static void createKey(OzoneBucket bucket, String key, String data) throws IOException {
+    byte[] dataBytes = data.getBytes(StandardCharsets.UTF_8);
+    try (OzoneOutputStream outputStream =
+             bucket.createKey(key, dataBytes.length, REPLICATION_CONFIG, new HashMap<>())) {
+      outputStream.write(dataBytes);
+      outputStream.flush();
+    }
+  }
+
+  public static String resourceAbsolutePath(String relativePath) {
+    return Optional.ofNullable(
+            OzoneClusterHarness.class.getClassLoader().getResource(relativePath))
+        .map(URL::getPath)
+        .orElseThrow(() -> new RuntimeException("Resource not found"));
+  }
+}

smart-ozone-support/smart-ozone-common/src/test/resources/docker-compose-ozone.yaml

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+x-image:
+  &image
+  image: ${OZONE_IMAGE:-apache/ozone}:${OZONE_IMAGE_VERSION:-2.0.0}${OZONE_IMAGE_FLAVOR:-}
+
+x-common-config:
+  &common-config
+  OZONE-SITE.XML_hdds.datanode.dir: "/data/hdds"
+  OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata"
+  OZONE-SITE.XML_ozone.om.address: "om"
+  OZONE-SITE.XML_ozone.administrators: "hadoop,om,hdfs,${ADMIN_USER:-root}"
+  OZONE-SITE.XML_ozone.om.http-address: "om:9874"
+  OZONE-SITE.XML_ozone.replication: "1"
+  OZONE-SITE.XML_ozone.scm.block.client.address: "scm"
+  OZONE-SITE.XML_ozone.scm.client.address: "scm"
+  OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data/metadata"
+  OZONE-SITE.XML_ozone.scm.names: "scm"
+  no_proxy: "om,scm,localhost,127.0.0.1"
+
+version: "3"
+services:
+  datanode:
+    <<: *image
+    ports:
+      - 9864
+    command: [ "ozone","datanode" ]
+    environment:
+      <<: *common-config
+
+  om:
+    <<: *image
+    ports:
+      - 9862
+    environment:
+      <<: *common-config
+      CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts: "*"
+      CORE-SITE.XML_hadoop.proxyuser.hadoop.groups: "*"
+      ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+      WAITFOR: scm:9876
+    command: [ "ozone","om" ]
+
+  scm:
+    <<: *image
+    ports:
+      - 9876
+    environment:
+      <<: *common-config
+      ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+    command: [ "ozone","scm" ]

smart-ozone-support/smart-ozone-fs-client/pom.xml

@@ -35,24 +35,14 @@
     <dependencies>
         <dependency>
             <groupId>org.smartdata</groupId>
-            <artifactId>smart-hadoop-common</artifactId>
+            <artifactId>smart-ozone-common</artifactId>
             <version>2.2.0-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>org.smartdata</groupId>
             <artifactId>smart-client</artifactId>
             <version>2.2.0-SNAPSHOT</version>
         </dependency>
-        <dependency>
-            <groupId>org.smartdata</groupId>
-            <artifactId>smart-common</artifactId>
-            <version>2.2.0-SNAPSHOT</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.ozone</groupId>
-            <artifactId>ozone-filesystem-hadoop3</artifactId>
-            <version>${ozone.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>
@@ -78,6 +68,18 @@
             <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.smartdata</groupId>
+            <artifactId>smart-ozone-common</artifactId>
+            <version>2.2.0-SNAPSHOT</version>
+            <scope>test</scope>
+            <type>test-jar</type>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

smart-ozone-support/smart-ozone-fs-client/src/main/java/org/smartdata/ozone/client/FileAccessReportSupport.java

@@ -44,7 +44,7 @@ public FileAccessReportSupport(SmartClientProtocol ssmClient, String... basePath
   }
 
   public void reportFileAccess(String path) {
-    String pathWithoutAuthority = new Path(basePath, path)
+    String pathWithoutAuthority = new Path(basePath, removeLeadingSlash(path))
         .toUri()
         .getPath();
 
@@ -60,6 +60,10 @@ public void reportFileAccess(String path) {
     }
   }
 
+  private String removeLeadingSlash(String path) {
+    return path.startsWith("/") ? path.substring(1) : path;
+  }
+
   @Override
   public void close() throws IOException {
     ssmClient.close();