HDDS-592. Fix ozone-secure.robot test. Contributed by Ajay Kumar.

hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot

@@ -1,95 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation       Smoke test to start cluster with docker-compose environments.
-Library             OperatingSystem
-Suite Setup         Startup Ozone Cluster
-Suite Teardown      Teardown Ozone Cluster
-
-*** Variables ***
-${COMMON_REST_HEADER}   -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H  "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root"
-${version}
-
-*** Test Cases ***
-
-Daemons are running
-    Is daemon running           om
-    Is daemon running           scm
-    Is daemon running           datanode
-    Is daemon running           ozone.kdc
-
-Check if datanode is connected to the scm
-    Wait Until Keyword Succeeds     3min    5sec    Have healthy datanodes   1
-
-Test rest interface
-    ${result} =     Execute on      0   datanode        curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
-                    Should contain      ${result}       201 Created
-    ${result} =     Execute on      0   datanode        curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
-                    Should contain      ${result}       201 Created
-    ${result} =     Execute on      0   datanode        curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
-                    Should contain      ${result}       200 OK
-    ${result} =     Execute on      0   datanode        curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
-                    Should contain      ${result}       200 OK
-
-Test ozone cli
-    ${result} =     Execute on      1   datanode        ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
-                    Should contain      ${result}       Client cannot authenticate via
-                    # Authenticate testuser
-                    Execute on      0   datanode        kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
-                    Execute on      0   datanode        ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
-    ${result} =     Execute on      0   datanode        ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
-                    Should contain      ${result}       createdOn
-                    Execute on      0   datanode        ozone oz -updateVolume o3://om/hive -user bill -quota 10TB
-    ${result} =     Execute on      0   datanode        ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
-                    Should Be Equal     ${result}       bill
-
-*** Keywords ***
-
-Startup Ozone Cluster
-    ${rc}       ${output} =                 Run docker compose        0     down
-    ${rc}       ${output} =                 Run docker compose        0     up -d
-    Should Be Equal As Integers 	          ${rc} 	                  0
-    Wait Until Keyword Succeeds             3min    10sec    Is Daemon started   ksm     KSM is listening
-
-Teardown Ozone Cluster
-    Run docker compose      0           down
-
-Is daemon running
-    [arguments]             ${name}
-    ${result} =             Run                     docker ps
-    Should contain          ${result}               _${name}_1
-
-Is Daemon started
-    [arguments]     ${name}             ${expression}
-    ${rc}           ${result} =         Run docker compose      0         logs
-    Should contain  ${result}           ${expression}
-
-Have healthy datanodes
-    [arguments]         ${requirednodes}
-    ${result} =         Execute on     0     scm                 curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value'
-    Should Be Equal     ${result}           ${requirednodes}
-
-Execute on
-    [arguments]     ${expected_rc}      ${componentname}      ${command}
-    ${rc}           ${return} =         Run docker compose    ${expected_rc}     exec ${componentname} ${command}
-    [return]        ${return}
-
-Run docker compose
-    [arguments]                     ${expected_rc}              ${command}
-                                    Set Environment Variable    OZONEDIR                               ${basedir}/hadoop-dist/target/ozone
-    ${rc}                           ${output} =                 Run And Return Rc And Output           docker-compose -f ${basedir}/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml ${command}
-    Should Be Equal As Integers     ${rc}                       ${expected_rc}
-    [return]                        ${rc}                       ${output}

hadoop-ozone/dist/src/main/compose/ozonesecure/.env

@@ -15,4 +15,3 @@
 # limitations under the License.
 
 HDDS_VERSION=${hdds.version}
-SRC_VOLUME=../../

hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml

@@ -24,7 +24,7 @@ services:
         buildno: 1
     hostname: kdc
     volumes:
-    - $SRC_VOLUME:/opt/hadoop
+      - ../..:/opt/hadoop
   datanode:
     build:
       context: docker-image/runner
@@ -32,13 +32,13 @@ services:
       args:
         buildno: 1
     volumes:
-    - $SRC_VOLUME:/opt/hadoop
+      - ../..:/opt/hadoop
     hostname: datanode
     ports:
-    - 9864
+      - 9864
     command: ["/opt/hadoop/bin/ozone","datanode"]
     env_file:
-    - docker-config
+      - docker-config
   om:
     build:
       context: docker-image/runner
@@ -47,13 +47,13 @@ services:
         buildno: 1
     hostname: om
     volumes:
-    - $SRC_VOLUME:/opt/hadoop
+      - ../..:/opt/hadoop
     ports:
-    - 9874:9874
+      - 9874:9874
     environment:
       ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
     env_file:
-    - docker-config
+      - docker-config
     command: ["/opt/hadoop/bin/ozone","om"]
   scm:
     build:
@@ -63,11 +63,11 @@ services:
         buildno: 1
     hostname: scm
     volumes:
-    - $SRC_VOLUME:/opt/hadoop
+      - ../..:/opt/hadoop
     ports:
-    - 9876:9876
+      - 9876:9876
     env_file:
-    - docker-config
+      - docker-config
     environment:
       ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
-    command: ["/opt/hadoop/bin/ozone","scm"]
+    command: ["/opt/hadoop/bin/ozone","scm"]

hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile

@@ -15,8 +15,8 @@
 # limitations under the License.
 
 FROM openjdk:8-jdk
-RUN apt-get update && apt-get install -y  jq curl python sudo && apt-get clean
-
+RUN apt-get update && apt-get install -y jq curl python python-pip sudo && apt-get clean
+RUN pip install robotframework
 RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
 RUN chmod +x /usr/local/bin/dumb-init
 RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs

hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot

@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Smoke test to start cluster with docker-compose environments.
+Library             OperatingSystem
+Resource            ../commonlib.robot
+
+*** Test Cases ***
+Create volume and bucket
+    ${rc}              ${output} =                 Run And Return Rc And Output              ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root
+                        Should contain       ${output}       Client cannot authenticate via
+                        # Authenticate testuser
+    Execute             kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
+    Execute             ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root
+    Execute             ozone sh volume create o3://om/fstest2 --user bilbo --quota 100TB --root
+    Execute             ozone sh bucket create o3://om/fstest/bucket1
+    Execute             ozone sh bucket create o3://om/fstest/bucket2
+    Execute             ozone sh bucket create o3://om/fstest2/bucket3
+
+Check volume from ozonefs
+    ${result} =         Execute               ozone fs -ls o3fs://bucket1.fstest/
+
+Run ozoneFS tests
+                        Execute               ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should contain    ${result}         testdir/deep
+                        Execute               ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should contain    ${result}         NOTICE.txt
+
+                        Execute               ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should contain    ${result}         PUTFILE.txt
+
+    ${result} =         Execute               ozone fs -ls o3fs://bucket1.fstest/testdir/deep/
+                        Should contain    ${result}         NOTICE.txt
+                        Should contain    ${result}         PUTFILE.txt
+
+                        Execute               ozone fs -mv o3fs://bucket1.fstest/testdir/deep/NOTICE.txt o3fs://bucket1.fstest/testdir/deep/MOVED.TXT
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should contain    ${result}         MOVED.TXT
+                        Should not contain  ${result}       NOTICE.txt
+
+                        Execute               ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep/subdir1
+                        Execute               ozone fs -cp o3fs://bucket1.fstest/testdir/deep/MOVED.TXT o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should contain    ${result}         subdir1/NOTICE.txt
+
+    ${result} =         Execute               ozone fs -ls o3fs://bucket1.fstest/testdir/deep/subdir1/
+                        Should contain    ${result}         NOTICE.txt
+
+                        Execute               ozone fs -cat o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+                        Should not contain  ${result}       Failed
+
+                        Execute               ozone fs -rm o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should not contain  ${result}       NOTICE.txt
+
+    ${result} =         Execute               ozone fs -rmdir o3fs://bucket1.fstest/testdir/deep/subdir1/
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should not contain  ${result}       subdir1
+
+                        Execute               ozone fs -touch o3fs://bucket1.fstest/testdir/TOUCHFILE.txt
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should contain  ${result}       TOUCHFILE.txt
+
+                        Execute               ozone fs -rm -r o3fs://bucket1.fstest/testdir/
+    ${result} =         Execute               ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                        Should not contain  ${result}       testdir
+
+                        Execute               rm -Rf localdir1
+                        Execute               mkdir localdir1
+                        Execute               cp NOTICE.txt localdir1/LOCAL.txt
+                        Execute               ozone fs -mkdir -p o3fs://bucket1.fstest/testdir1
+                        Execute               ozone fs -copyFromLocal localdir1 o3fs://bucket1.fstest/testdir1/
+                        Execute               ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir1/NOTICE.txt
+
+    ${result} =         Execute               ozone fs -ls -R o3fs://bucket1.fstest/testdir1/
+                        Should contain    ${result}         localdir1/LOCAL.txt
+                        Should contain    ${result}         testdir1/NOTICE.txt
+
+                        Execute               ozone fs -mkdir -p o3fs://bucket2.fstest/testdir2
+                        Execute               ozone fs -mkdir -p o3fs://bucket3.fstest2/testdir3
+                        Execute               ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket2.fstest/testdir2/
+                        Execute               ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket3.fstest2/testdir3/
+                        Execute               ozone sh key put o3://om/fstest/bucket1/KEY.txt NOTICE.txt
+    ${result} =         Execute               ozone fs -ls o3fs://bucket1.fstest/KEY.txt
+                        Should contain    ${result}         KEY.txt
+    ${rc}  ${result} =  Run And Return Rc And Output        ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/KEY.txt
+                        Should Be Equal As Integers     ${rc}                1
+                        Should contain    ${result}         File exists
+                        Execute               rm -Rf GET.txt
+                        Execute               ozone fs -get o3fs://bucket1.fstest/KEY.txt GET.txt
+                        Execute               ls -l GET.txt
+    ${rc}  ${result} =  Run And Return Rc And Output        ozone fs -ls o3fs://abcde.pqrs/
+                        Should Be Equal As Integers     ${rc}                1
+                        Should contain    ${result}         VOLUME_NOT_FOUND
+

hadoop-ozone/dist/src/main/smoketest/test.sh

@@ -107,6 +107,8 @@ if [ "$RUN_ALL" = true ]; then
 #
 # We select the test suites and execute them on multiple type of clusters
 #
+   DEFAULT_TESTS=("security")
+   execute_tests ozonesecure "${DEFAULT_TESTS[@]}"
    DEFAULT_TESTS=("basic")
    execute_tests ozone "${DEFAULT_TESTS[@]}"
    TESTS=("ozonefs")