HDDS-1925. ozonesecure acceptance test broken by HTTP auth requirement (#1248)

hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh

@@ -33,7 +33,7 @@ execute_robot_test scm security
 
 execute_robot_test scm ozonefs/ozonefs.robot
 
-execute_robot_test scm s3
+execute_robot_test s3g s3
 
 stop_docker_env
 

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -28,6 +28,20 @@ mkdir -p "$RESULT_DIR"
 #Should be writeable from the docker containers where user is different.
 chmod ogu+w "$RESULT_DIR"
 
+## @description print the number of datanodes up
+## @param the docker-compose file
+count_datanodes() {
+  local compose_file=$1
+
+  local jmx_url='http://scm:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo'
+  if [[ "${SECURITY_ENABLED}" == 'true' ]]; then
+    docker-compose -f "${compose_file}" exec -T scm bash -c "kinit -k HTTP/scm@EXAMPLE.COM -t /etc/security/keytabs/HTTP.keytab && curl --negotiate -u : -s '${jmx_url}'"
+  else
+    docker-compose -f "${compose_file}" exec -T scm curl -s "${jmx_url}"
+  fi \
+    | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value'
+}
+
 ## @description wait until datanodes are up (or 30 seconds)
 ## @param the docker-compose file
 ## @param number of datanodes to wait for (default: 3)
@@ -43,7 +57,7 @@ wait_for_datanodes(){
 
      #This line checks the number of HEALTHY datanodes registered in scm over the
      # jmx HTTP servlet
-     datanodes=$(docker-compose -f "${compose_file}" exec -T scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value')
+     datanodes=$(count_datanodes "${compose_file}")
      if [[ "$datanodes" ]]; then
        if [[ ${datanodes} -ge ${datanode_count} ]]; then
 

hadoop-ozone/dist/src/main/smoketest/basic/basic.robot

@@ -25,9 +25,8 @@ ${DATANODE_HOST}        datanode
 *** Test Cases ***
 
 Check webui static resources
-    ${result} =        Execute                curl -s -I http://scm:9876/static/bootstrap-3.3.7/js/bootstrap.min.js
-                       Should contain         ${result}    200
-    ${result} =        Execute                curl -s -I http://om:9874/static/bootstrap-3.3.7/js/bootstrap.min.js
+    Run Keyword if    '${SECURITY_ENABLED}' == 'true'    Kinit HTTP user
+    ${result} =        Execute                curl --negotiate -u : -s -I http://scm:9876/static/bootstrap-3.3.7/js/bootstrap.min.js
                        Should contain         ${result}    200
 
 Start freon testing

hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot

@@ -17,6 +17,7 @@
 Documentation       Test ozone shell CLI usage
 Library             OperatingSystem
 Resource            ../commonlib.robot
+Test Setup          Run Keyword if    '${SECURITY_ENABLED}' == 'true'    Kinit test user     testuser     testuser.keytab
 Test Timeout        2 minute
 
 *** Variables ***

hadoop-ozone/dist/src/main/smoketest/commonlib.robot

@@ -54,8 +54,12 @@ Install aws cli
     ${rc}              ${output} =                 Run And Return Rc And Output           yum --help
     Run Keyword if     '${rc}' == '0'              Install aws cli s3 centos
 
+Kinit HTTP user
+    ${hostname} =       Execute                    hostname
+    Wait Until Keyword Succeeds      2min       10sec      Execute            kinit -k HTTP/${hostname}@EXAMPLE.COM -t /etc/security/keytabs/HTTP.keytab
+
 Kinit test user
     [arguments]                      ${user}       ${keytab}
     ${hostname} =       Execute                    hostname
     Set Suite Variable  ${TEST_USER}               ${user}/${hostname}@EXAMPLE.COM
-    Wait Until Keyword Succeeds      2min       10sec      Execute            kinit -k ${user}/${hostname}@EXAMPLE.COM -t /etc/security/keytabs/${keytab}
+    Wait Until Keyword Succeeds      2min       10sec      Execute            kinit -k ${user}/${hostname}@EXAMPLE.COM -t /etc/security/keytabs/${keytab}

hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot

@@ -49,6 +49,7 @@ Setup v2 headers
                         Set Environment Variable   AWS_SECRET_ACCESS_KEY   ANYKEY
 
 Setup v4 headers
+    Run Keyword if      '${SECURITY_ENABLED}' == 'true'     Kinit test user    testuser    testuser.keytab
     ${result} =         Execute                    ozone s3 getsecret
     ${accessKey} =      Get Regexp Matches         ${result}     (?<=awsAccessKey=).*
     ${accessKey} =      Get Variable Value         ${accessKey}  sdsdasaasdasd

hadoop-ozone/dist/src/main/smoketest/s3/webui.robot

@@ -27,8 +27,9 @@ ${BUCKET}             generated
 
 *** Test Cases ***
 
-File upload and directory list
-    ${result} =         Execute                             curl -v ${ENDPOINT_URL}
+S3 Gateway Web UI
+    Run Keyword if      '${SECURITY_ENABLED}' == 'true'     Kinit HTTP user
+    ${result} =         Execute                             curl --negotiate -u : -v ${ENDPOINT_URL}
                         Should contain      ${result}       HTTP/1.1 307 Temporary Redirect
-    ${result} =         Execute                             curl -v ${ENDPOINT_URL}/static/
+    ${result} =         Execute                             curl --negotiate -u : -v ${ENDPOINT_URL}/static/index.html
                         Should contain      ${result}       Apache Hadoop Ozone S3