HDDS-177. Create a releasable ozonefs artifact

Contributed by Marton, Elek.

dev-support/bin/ozone-dist-layout-stitching

@@ -145,6 +145,8 @@ run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-$
 run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}" .
 run copy "${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" .
 run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" .
+mkdir -p "./share/hadoop/ozonefs"
+cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem.jar"
 # Optional documentation, could be missing
 cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ksm/
 cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/
@@ -153,5 +155,5 @@ cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdd
 mkdir -p ./share/hadoop/mapreduce
 mkdir -p ./share/hadoop/yarn
 echo
-echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone-${HDDS_VERSION}"
+echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone"
 echo

hadoop-dist/pom.xml

@@ -265,6 +265,11 @@
           <artifactId>hadoop-ozone-docs</artifactId>
           <scope>provided</scope>
         </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-ozone-filesystem</artifactId>
+          <scope>provided</scope>
+        </dependency>
       </dependencies>
       <build>
         <plugins>

hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml

@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+   namenode:
+      image: apache/hadoop-runner
+      hostname: namenode
+      volumes:
+         - ${OZONEDIR}:/opt/hadoop
+      ports:
+         - 9870
+      environment:
+          ENSURE_NAMENODE_DIR: /data/namenode
+      env_file:
+         - ./docker-config
+      command: ["/opt/hadoop/bin/hdfs","namenode"]
+   datanode:
+      image: apache/hadoop-runner
+      volumes:
+        - ${OZONEDIR}:/opt/hadoop
+      ports:
+        - 9864
+      command: ["/opt/hadoop/bin/ozone","datanode"]
+      env_file:
+        - ./docker-config
+   ksm:
+      image: apache/hadoop-runner
+      hostname: ksm
+      volumes:
+         - ${OZONEDIR}:/opt/hadoop
+      ports:
+         - 9874
+      environment:
+         ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
+      env_file:
+          - ./docker-config
+      command: ["/opt/hadoop/bin/ozone","ksm"]
+   scm:
+      image: apache/hadoop-runner
+      volumes:
+         - ${OZONEDIR}:/opt/hadoop
+      ports:
+         - 9876
+      env_file:
+          - ./docker-config
+      environment:
+          ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+      command: ["/opt/hadoop/bin/ozone","scm"]
+   hadooplast:
+      image: flokkr/hadoop:3.1.0
+      volumes:
+        - ${OZONEDIR}:/opt/ozone
+      env_file:
+        - ./docker-config
+      environment:
+         HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/hadoop-ozone-filesystem.jar
+      command: ["watch","-n","100000","ls"]

hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
+CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
+OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.category.org.apache.hadoop.util.NativeCodeLoader=ERROR

hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Ozonefs test
+Library             OperatingSystem
+Suite Setup         Startup Ozone cluster with size          5
+Suite Teardown      Teardown Ozone cluster
+Resource            ../commonlib.robot
+
+*** Variables ***
+${COMPOSEFILE}          ${CURDIR}/docker-compose.yaml
+${PROJECTDIR}           ${CURDIR}/../../../../../..
+
+
+*** Test Cases ***
+Create volume and bucket
+    Execute on          datanode        ozone oz -createVolume http://ksm/fstest -user bilbo -quota 100TB -root
+    Execute on          datanode        ozone oz -createBucket http://ksm/fstest/bucket1
+
+Check volume from ozonefs
+    ${result} =         Execute on          hadooplast        hdfs dfs -ls o3://bucket1.fstest/
+
+Create directory from ozonefs
+                        Execute on          hadooplast        hdfs dfs -mkdir -p o3://bucket1.fstest/testdir/deep
+    ${result} =         Execute on          ksm               ozone oz -listKey o3://ksm/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                                            Should contain    ${result}         testdir/deep

hadoop-tools/hadoop-ozone/pom.xml → hadoop-ozone/ozonefs/pom.xml

@@ -18,14 +18,13 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.hadoop</groupId>
-    <artifactId>hadoop-project</artifactId>
-    <version>3.2.0-SNAPSHOT</version>
-    <relativePath>../../hadoop-project</relativePath>
+    <artifactId>hadoop-ozone</artifactId>
+    <version>0.2.1-SNAPSHOT</version>
   </parent>
   <artifactId>hadoop-ozone-filesystem</artifactId>
   <name>Apache Hadoop Ozone FileSystem</name>
   <packaging>jar</packaging>
-
+  <version>0.2.1-SNAPSHOT</version>
   <properties>
     <file.encoding>UTF-8</file.encoding>
     <downloadSources>true</downloadSources>
@@ -44,6 +43,46 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>3.1.1</version>
+        <configuration>
+          <artifactSet>
+            <includes>
+              <include>com.google.guava:guava:jar</include>
+              <include>org.slf4j:slf4j-api:jar</include>
+              <include>com.google.protobuf:protobuf-java</include>
+              <include>com.nimbusds:nimbus-jose-jwt:jar</include>
+              <include>com.github.stephenc.jcip:jcip-annotations</include>
+              <include>com.google.code.findbugs:jsr305:jar</include>
+              <include>org.apache.hadoop:hadoop-ozone-client</include>
+              <include>org.apache.hadoop:hadoop-hdds-client</include>
+              <include>org.apache.hadoop:hadoop-hdds-common</include>
+              <include>org.fusesource.leveldbjni:leveldbjni-all</include>
+              <include>org.apache.ratis:ratis-server</include>
+              <include>org.apache.ratis:ratis-proto-shaded:jar</include>
+              <include>com.google.auto.value:auto-value-annotations</include>
+              <include>com.squareup:javapoet:jar</include>
+              <include>org.jctools:jctools-core</include>
+              <include>org.apache.ratis:ratis-common</include>
+              <include>org.apache.ratis:ratis-client</include>
+              <include>org.apache.ratis:ratis-netty</include>
+              <include>org.apache.ratis:ratis-grpc</include>
+              <include>org.rocksdb:rocksdbjni</include>
+              <include>org.apache.hadoop:hadoop-ozone-common</include>
+            </includes>
+          </artifactSet>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
@@ -56,7 +95,9 @@
             </goals>
             <configuration>
               <!-- build a shellprofile -->
-              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
+              <outputFile>
+                ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt
+              </outputFile>
             </configuration>
           </execution>
         </executions>
@@ -83,7 +124,6 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdds-common</artifactId>
-      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -108,12 +148,10 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdds-client</artifactId>
-      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-ozone-common</artifactId>
-      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -129,7 +167,6 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-ozone-client</artifactId>
-      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>

hadoop-ozone/pom.xml

@@ -34,6 +34,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <module>client</module>
     <module>ozone-manager</module>
     <module>tools</module>
+    <module>ozonefs</module>
     <module>integration-test</module>
     <module>objectstore-service</module>
     <module>docs</module>

hadoop-project/pom.xml

@@ -586,7 +586,11 @@
         <artifactId>hadoop-ozone-docs</artifactId>
         <version>${hdds.version}</version>
       </dependency>
-
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-ozone-filesystem</artifactId>
+        <version>${hdds.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-hdds-common</artifactId>

hadoop-tools/hadoop-tools-dist/pom.xml

@@ -192,20 +192,5 @@
         </plugins>
       </build>
     </profile>
-
-    <profile>
-      <id>hdds</id>
-      <activation>
-        <activeByDefault>false</activeByDefault>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-ozone-filesystem</artifactId>
-          <scope>compile</scope>
-          <version>${project.version}</version>
-        </dependency>
-      </dependencies>
-    </profile>
   </profiles>
 </project>

hadoop-tools/pom.xml

@@ -67,15 +67,4 @@
       </plugin>
     </plugins>
   </build>
-  <profiles>
-    <profile>
-      <id>hdds</id>
-      <activation>
-        <activeByDefault>false</activeByDefault>
-      </activation>
-      <modules>
-        <module>hadoop-ozone</module>
-      </modules>
-    </profile>
-  </profiles>
 </project>