HADOOP-17191. ABFS: Run the tests with various combinations of configurations and publish a consolidated results

- Contributed by Bilahari T H

hadoop-tools/hadoop-azure/.gitignore

@@ -1,2 +1,5 @@
 .checkstyle
-bin/
+bin/
+src/test/resources/combinationConfigFiles
+src/test/resources/abfs-combination-test-configs.xml
+dev-support/testlogs

hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2034
+# unused variables are global in nature and used in testsupport.sh
+
+set -eo pipefail
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# shellcheck disable=SC1091
+. dev-support/testrun-scripts/testsupport.sh
+
+begin
+
+### ADD THE TEST COMBINATIONS BELOW. DO NOT EDIT THE ABOVE LINES.
+
+
+combination=HNS-OAuth
+properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
+"fs.azure.account.auth.type")
+values=("{account name}.dfs.core.windows.net" "true" "OAuth")
+generateconfigs
+
+combination=HNS-SharedKey
+properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
+values=("{account name}.dfs.core.windows.net" "true" "SharedKey")
+generateconfigs
+
+combination=NonHNS-SharedKey
+properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
+values=("{account name}.dfs.core.windows.net" "false" "SharedKey")
+generateconfigs
+
+
+### DO NOT EDIT THE LINES BELOW.
+
+runtests "$@"

hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh

@@ -0,0 +1,241 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+testresourcesdir=src/test/resources
+combconfsdir=$testresourcesdir/combinationConfigFiles
+combtestfile=$testresourcesdir/abfs-combination-test-configs.xml
+
+logdir=dev-support/testlogs
+testresultsregex="Results:(\n|.)*?Tests run:"
+testresultsfilename=
+starttime=
+threadcount=
+defaultthreadcount=8
+
+properties=
+values=
+
+validate() {
+  if [ -z "$threadcount" ] ; then
+    threadcount=$defaultthreadcount
+  fi
+  numberegex='^[0-9]+$'
+  if ! [[ $threadcount =~ $numberegex ]] ; then
+    echo "Exiting. The script param (threadcount) should be a number"
+    exit -1
+  fi
+  if [ -z "$combination" ]; then
+   echo "Exiting. combination cannot be empty"
+   exit -1
+  fi
+  propertiessize=${#properties[@]}
+  valuessize=${#values[@]}
+  if [ "$propertiessize" -lt 1 ] || [ "$valuessize" -lt 1 ] || [ "$propertiessize" -ne "$valuessize" ]; then
+    echo "Exiting. Both properties and values arrays has to be populated and of same size. Please check for combination $combination"
+    exit -1
+  fi
+
+  for filename in "${combinations[@]}"; do
+    if [[ ! -f "$combconfsdir/$filename.xml" ]]; then
+      echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
+      exit -1
+    fi
+  done
+}
+
+checkdependencies() {
+  if ! [ "$(command -v pcregrep)" ]; then
+    echo "Exiting. pcregrep is required to run the script."
+    exit -1
+  fi
+  if ! [ "$(command -v xmlstarlet)" ]; then
+    echo "Exiting. xmlstarlet is required to run the script."
+    exit -1
+  fi
+}
+
+cleancombinationconfigs() {
+  rm -rf $combconfsdir
+  mkdir -p $combconfsdir
+}
+
+generateconfigs() {
+  combconffile="$combconfsdir/$combination.xml"
+  rm -rf "$combconffile"
+  cat > "$combconffile" << ENDOFFILE
+<configuration>
+
+</configuration>
+ENDOFFILE
+
+  propertiessize=${#properties[@]}
+  valuessize=${#values[@]}
+  if [ "$propertiessize" -ne "$valuessize" ]; then
+    echo "Exiting. Number of properties and values differ for $combination"
+    exit -1
+  fi
+  for ((i = 0; i < propertiessize; i++)); do
+    key=${properties[$i]}
+    val=${values[$i]}
+    changeconf "$key" "$val"
+  done
+  formatxml "$combconffile"
+}
+
+formatxml() {
+  xmlstarlet fo -s 2 "$1" > "$1.tmp"
+  mv "$1.tmp" "$1"
+}
+
+setactiveconf() {
+  if [[ ! -f "$combconfsdir/$combination.xml" ]]; then
+    echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
+    exit -1
+  fi
+  rm -rf $combtestfile
+  cat > $combtestfile << ENDOFFILE
+<configuration>
+
+</configuration>
+ENDOFFILE
+  xmlstarlet ed -P -L -s /configuration -t elem -n include -v "" $combtestfile
+  xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "combinationConfigFiles/$combination.xml" $combtestfile
+  xmlstarlet ed -P -L -i /configuration/include -t attr -n xmlns -v "http://www.w3.org/2001/XInclude" $combtestfile
+  formatxml $combtestfile
+}
+
+changeconf() {
+  xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combconffile"
+  xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combconffile"
+  if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combconffile"
+  then
+    echo "Exiting. Changing config property failed."
+    exit -1
+  fi
+}
+
+summary() {
+  {
+    echo ""
+    echo "$combination"
+    echo "========================"
+    pcregrep -M "$testresultsregex" "$testlogfilename"
+  } >> "$testresultsfilename"
+  printf "\n----- Test results -----\n"
+  pcregrep -M "$testresultsregex" "$testlogfilename"
+
+  secondstaken=$((ENDTIME - STARTTIME))
+  mins=$((secondstaken / 60))
+  secs=$((secondstaken % 60))
+  printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
+  echo "Find test logs for the combination ($combination) in: $testlogfilename"
+  echo "Find consolidated test results in: $testresultsfilename"
+  echo "----------"
+}
+
+init() {
+  checkdependencies
+  if ! mvn clean install -DskipTests
+  then
+    echo ""
+    echo "Exiting. Build failed."
+    exit -1
+  fi
+  starttime=$(date +"%Y-%m-%d_%H-%M-%S")
+  mkdir -p "$logdir"
+  testresultsfilename="$logdir/$starttime/Test-Results.txt"
+  if [[ -z "$combinations" ]]; then
+    combinations=( $( ls $combconfsdir/*.xml ))
+  fi
+}
+
+runtests() {
+  parseoptions "$@"
+  validate
+  if [ -z "$starttime" ]; then
+    init
+  fi
+  shopt -s nullglob
+  for combconffile in "${combinations[@]}"; do
+    STARTTIME=$(date +%s)
+    combination=$(basename "$combconffile" .xml)
+    mkdir -p "$logdir/$starttime"
+    testlogfilename="$logdir/$starttime/Test-Logs-$combination.txt"
+    printf "\nRunning the combination: %s..." "$combination"
+    setactiveconf
+    mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=$threadcount verify >> "$testlogfilename" || true
+    ENDTIME=$(date +%s)
+    summary
+  done
+}
+
+begin() {
+  cleancombinationconfigs
+}
+
+parseoptions() {
+runactivate=0
+runtests=0
+  while getopts ":c:a:t:" option; do
+    case "${option}" in
+      a)
+        if [[ "$runactivate" -eq "1" ]]; then
+          echo "-a Option is not multivalued"
+          exit 1
+        fi
+        runactivate=1
+        combination=$(basename "$OPTARG" .xml)
+        ;;
+      c)
+        runtests=1
+        combination=$(basename "$OPTARG" .xml)
+        combinations+=("$combination")
+        ;;
+      t)
+        threadcount=$OPTARG
+        ;;
+      *|?|h)
+        if [[ -z "$combinations" ]]; then
+          combinations=( $( ls $combconfsdir/*.xml ))
+        fi
+      combstr=""
+        for combconffile in "${combinations[@]}"; do
+          combname=$(basename "$combconffile" .xml)
+          combstr="${combname}, ${combstr}"
+        done
+        combstr=${combstr:0:-2}
+
+        echo "Usage: $0 [-n] [-a COMBINATION_NAME] [-c COMBINATION_NAME] [-t THREAD_COUNT]"
+        echo ""
+        echo "Where:"
+        echo "  -a COMBINATION_NAME   Specify the combination name which needs to be activated."
+        echo "                        Configured combinations: ${combstr}"
+        echo "  -c COMBINATION_NAME   Specify the combination name for test runs"
+        echo "  -t THREAD_COUNT       Specify the thread count"
+        exit 1
+        ;;
+    esac
+  done
+  if [[ "$runactivate" -eq "1" && "$runtests" -eq "1" ]]; then
+    echo "Both activate (-a option) and test run combinations (-c option) cannot be specified together"
+    exit 1
+  fi
+  if [[ "$runactivate" -eq "1" ]]; then
+        setactiveconf
+        exit 0
+  fi
+}

hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md

@@ -592,6 +592,61 @@ with the Hadoop Distributed File System permissions model when hierarchical
 namespace is enabled for the storage account.  Furthermore, the metadata and data
 produced by ADLS Gen 2 REST API can be consumed by Blob REST API, and vice versa.
 
+## Generating test run configurations and test triggers over various config combinations
+
+To simplify the testing across various authentication and features combinations
+that are mandatory for a PR, script `dev-support/testrun-scripts/runtests.sh`
+should be used. Once the script is updated with relevant config settings for
+various test combinations, it will:
+1. Auto-generate configs specific to each test combinations
+2. Run tests for all combinations
+3. Summarize results across all the test combination runs.
+
+As a pre-requiste step, fill config values for test accounts and credentials
+needed for authentication in `src/test/resources/azure-auth-keys.xml.template`
+and rename as `src/test/resources/azure-auth-keys.xml`.
+
+**To add a new test combination:** Templates for mandatory test combinations
+for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
+If a new one needs to be added, add a combination set within
+`dev-support/testrun-scripts/runtests.sh` similar to the ones already defined
+and
+1. Provide a new combination name
+2. Update properties and values array which need to be effective for the test
+combination
+3. Call generateconfigs
+
+**To run PR validation:** Running command
+* `dev-support/testrun-scripts/runtests.sh` will generate configurations for
+each of the combinations defined and run tests for all the combinations.
+* `dev-support/testrun-scripts/runtests.sh -c {combinationname}` Specific
+combinations can be provided with -c option. If combinations are provided
+with -c option, tests for only those combinations will be run.
+
+**Test logs:** Test runs will create a folder within dev-support/testlogs to
+save the test logs. Folder name will be the test start timestamp. The mvn verify
+command line logs for each combination will be saved into a file as
+Test-Logs-$combination.txt into this folder. In case of any failures, this file
+will have the failure exception stack. At the end of the test run, the
+consolidated results of all the combination runs will be saved into a file as
+Test-Results.log in the same folder. When run for PR validation, the
+consolidated test results needs to be pasted into the PR comment section.
+
+**To generate config for use in IDE:** Running command with -a (activate) option
+`dev-support/testrun-scripts/runtests.sh -a {combination name}` will update
+the effective config relevant for the specific test combination. Hence the same
+config files used by the mvn test runs can be used for IDE without any manual
+updates needed within config file.
+
+**Other command line options:**
+* -a <COMBINATION_NAME> Specify the combination name which needs to be
+activated. This is to be used to generate config for use in IDE.
+* -c <COMBINATION_NAME> Specify the combination name for test runs. If this
+config is specified, tests for only the specified combinations will run. All
+combinations of tests will be running if this config is not specified.
+* -t <THREAD_COUNT> ABFS mvn tests are run in parallel mode. Tests by default
+are run with 8 thread count. It can be changed by providing -t <THREAD_COUNT>
+
 In order to test ABFS, please add the following configuration to your
 `src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include
 compatibility tests which require WASB credentials, in addition to the ABFS

hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template

@@ -0,0 +1,174 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<configuration>
+
+  <!--====================== IMPORTANT!! ========================-->
+  <!--
+      1. COPY THE CONTENTS OF THIS FILE TO "azure-auth-keys.xml" AND THEN EDIT.
+      2. UPDATE runtests.sh
+         A.  "fs.azure.abfs.account.name" AND "fs.azure.test.namespace.enabled"
+             ARE MANDATORY WITH EVERY SCENARIO. AUTHTYPE BY DEFAULT WILL BE
+             SHAREDKEY AS CONFIGURED IN THIS TEMPLATE FILE.
+         B.  PLEASE ADD MORE SCENARIOS IF THE CODE CHANGE REQUIRES TESTING WITH
+             DIFFERENT VARIANTS OF CONFIGS.
+      3. THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep
+      4. NOW THE SCRIPT CAN BE EXECUTED WITH ./runtests.sh
+  -->
+
+  <!--=============== Auth type ===============-->
+  <property>
+    <name>fs.azure.account.auth.type</name>
+    <value>SharedKey</value>
+  </property>
+
+  <!--=============== Auth related accounts ===============-->
+  <!-- This set of configs needs to be provided for all the accounts with which
+   the tests needs to be ran. -->
+  <property>
+    <name>fs.azure.account.key.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net</name>
+    <value>{ACCOUNT_ACCESS_KEY}</value>
+    <description>Account access key</description>
+  </property>
+  <property>
+    <name>
+      fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
+    </name>
+    <value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
+    <description>OAuth token provider implementation class</description>
+  </property>
+  <property>
+    <name>
+      fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
+    </name>
+    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
+    <description>Token end point, this can be found through Azure portal
+    </description>
+  </property>
+  <property>
+    <name>
+      fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
+    </name>
+    <value>{client id}</value>
+    <description>AAD client id.</description>
+  </property>
+  <property>
+    <name>
+      fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
+    </name>
+    <value>{client secret}</value>
+    <description>AAD client secret</description>
+  </property>
+
+  <!--=============== Configs for hadoop contract tests for ABFS ===============-->
+  <property>
+    <name>fs.contract.test.fs.abfs</name>
+    <value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.contract.test.fs.abfss</name>
+    <value>abfss://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
+  </property>
+
+  <!--=============== WASB Configs ===============-->
+  <property>
+    <name>fs.azure.wasb.account.name</name>
+    <value>{WASB_ACCOUNT_NAME}.blob.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.azure.account.key.{WASB_ACCOUNT_NAME}.blob.core.windows.net</name>
+    <value>WASB account key</value>
+  </property>
+  <property>
+    <name>fs.contract.test.fs.wasb</name>
+    <value>wasb://{WASB_FILESYSTEM}@{WASB_ACCOUNT_NAME}.blob.core.windows.net
+    </value>
+  </property>
+
+  <!--============= Configs for ITestAzureBlobFileSystemOauth tests ===============-->
+  <property>
+    <name>fs.azure.account.oauth2.contributor.client.id</name>
+    <value>{Client id of SP with RBAC Storage Blob Data Contributor}</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.contributor.client.secret</name>
+    <value>{Client secret of SP with RBAC Storage Blob Data Contributor}</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.reader.client.id</name>
+    <value>{Client id of SP with RBAC Storage Blob Data Reader}</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.reader.client.secret</name>
+    <value>{Client secret of SP with RBAC Storage Blob Data Reader}</value>
+  </property>
+
+<!--===========================   FOR CheckAccess =========================-->
+<!-- To run ABFS CheckAccess tests, you must register an app, with no role
+ assignments, and set the configuration discussed below:
+
+    1) Register a new app with no RBAC
+    2) As part of the test configs you need to provide the guid for the above
+created app. Please follow the below steps to fetch the guid.
+      a) Get an access token with the above created app. Please refer the
+ following documentation for the same. https://docs.microsoft
+.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow#get-a-token
+      b) Decode the token fetched with the above step. You may use https
+://jwt.ms/ to decode the token
+      d) The oid field in the decoded string is the guid.
+    3) Set the following configurations:
+-->
+
+  <property>
+    <name>fs.azure.account.test.oauth2.client.id</name>
+    <value>{client id}</value>
+    <description>The client id(app id) for the app created on step 1
+    </description>
+  </property>
+  <property>
+    <name>fs.azure.account.test.oauth2.client.secret</name>
+    <value>{client secret}</value>
+    <description>
+The client secret(application's secret) for the app created on step 1
+    </description>
+  </property>
+  <property>
+    <name>fs.azure.check.access.testuser.guid</name>
+    <value>{guid}</value>
+    <description>The guid fetched on step 2</description>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.client.endpoint.{account name}.dfs.core
+.windows.net</name>
+    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
+    <description>
+Token end point. This can be found through Azure portal. As part of CheckAccess
+test cases. The access will be tested for an FS instance created with the
+above mentioned client credentials. So this configuration is necessary to
+create the test FS instance.
+    </description>
+  </property>
+
+  <!--========== Append blob related configs ===========-->
+  <property>
+    <name>fs.azure.test.appendblob.enabled</name>
+    <value>false</value>
+    <description>If made true, tests will be running under the assumption that
+      append blob is enabled and the root directory and contract test root
+      directory will be part of the append blob directories.
+    </description>
+  </property>
+
+</configuration>

hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml

@@ -66,4 +66,8 @@
     <fallback />
   </include>
 
+  <include xmlns="http://www.w3.org/2001/XInclude" href="abfs-combination-test-configs.xml">
+    <fallback />
+  </include>
+
 </configuration>