HADOOP-12101. Add automatic search of default Configuration variables to TestConfigurationFieldsBase. Contributed by Ray Chiang.

(cherry picked from commit 355325bcc7111fa4aac801fd23a26422ffabaf7c)

dev-support/verify-xml.sh

@@ -0,0 +1,150 @@
+#!/bin/bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+# Script to run unit tests for xml <-> 1 or more Configuration file verification
+# usage: ./verify-xml.sh <mode>
+#
+
+# Utility functions
+function find_test_output_file() {
+  echo "Found test output file(s) at"
+  echo ""
+  if [ -n "$1" ] && [ -e "$1" ] ; then
+    echo "  $1"
+  fi
+  if [ -n "$2" ] && [ -e "$2" ] ; then
+    echo "  $2"
+  fi
+  if [ -n "$3" ] && [ -e "$3" ] ; then
+    echo "  $3"
+  fi
+  if [ -n "$4" ] && [ -e "$4" ] ; then
+    echo "  $4"
+  fi
+  echo ""
+  echo "Examine the file for specific information xml/Configuration mismatches."
+  echo ""
+}
+
+function print_test_banner() {
+  local banner_text=$1
+  local banner_length=${#banner_text}
+  local banner
+  banner=$( printf "%${banner_length}s" ' ' )
+  echo ""
+  echo "${banner// /=}"
+  echo "${banner_text}"
+  echo "${banner// /=}"
+  echo ""
+}
+
+# Wrapper functions for running unit tests
+function run_all_xml_test() {
+  mvn test -Dtest=TestCommonConfigurationFields,TestHdfsConfigFields,TestMapreduceConfigFields,TestYarnConfigurationFields
+  if [ $? -ne 0 ] ; then
+    print_test_banner "All Test*ConfigFields FAIL"
+  else
+    print_test_banner "All Test*ConfigFields SUCCESS"
+  fi
+}
+
+function run_common_xml_test() {
+  mvn test -Dtest=TestCommonConfigFields
+  if [ $? -ne 0 ] ; then
+    print_test_banner "TestCommonConfigurationFields FAIL"
+  else
+    print_test_banner "TestCommonConfigurationFields SUCCESS"
+  fi
+}
+
+function run_hdfs_xml_test() {
+  mvn test -Dtest=TestHdfsConfigFields
+  if [ $? -ne 0 ] ; then
+    print_test_banner "TestHdfsConfigFields FAIL"
+  else
+    print_test_banner "TestHdfsConfigFields SUCCESS"
+  fi
+}
+
+function run_mapreduce_xml_test() {
+  mvn test -Dtest=TestMapreduceConfigFields
+  if [ $? -ne 0 ] ; then
+    print_test_banner "TestMapreduceConfigFields FAIL"
+  else
+    print_test_banner "TestMapreduceConfigFields SUCCESS"
+  fi
+}
+
+function run_yarn_xml_test() {
+  mvn test -Dtest=TestYarnConfigurationFields
+  if [ $? -ne 0 ] ; then
+    print_test_banner "TestYarnConfigurationFields FAIL"
+  else
+    print_test_banner "TestYarnConfigurationFields SUCCESS"
+  fi
+}
+
+# Main body
+cd -P -- "$(dirname -- "${BASH_SOURCE-$0}")/.." || exit
+dir="$(pwd -P)"
+
+# - Create unit test file names
+export commonOutputFile
+commonOutputFile="$(find "${dir}" -name org.apache.hadoop.conf.TestCommonConfigurationFields-output.txt)"
+export hdfsOutputFile
+hdfsOutputFile="$(find "${dir}" -name org.apache.hadoop.tools.TestHdfsConfigFields-output.txt)"
+export mrOutputFile
+mrOutputFile="$(find "${dir}" -name org.apache.hadoop.mapreduce.TestMapreduceConfigFields-output.txt)"
+export yarnOutputFile
+yarnOutputFile="$(find "${dir}" -name org.apache.hadoop.yarn.conf.TestYarnConfigurationFields-output.txt)"
+
+# - Determine which tests to run
+case "$1" in
+
+  all)
+    run_all_xml_test
+    find_test_output_file "${commonOutputFile}" "${hdfsOutputFile}" "${mrOutputFile}" "${yarnOutputFile}"
+    ;;
+
+  common)
+    run_common_xml_test
+    find_test_output_file "${commonOutputFile}"
+    ;;
+
+  hdfs)
+    run_hdfs_xml_test
+    find_test_output_file "${hdfsOutputFile}"
+    ;;
+
+  mr)
+    run_mapreduce_xml_test
+    find_test_output_file "${mrOutputFile}"
+    ;;
+
+  yarn)
+    run_yarn_xml_test
+    find_test_output_file "${yarnOutputFile}"
+    ;;
+
+  *)
+    echo "Usage: $0 <mode>"
+    echo "  where <mode> is one of all, common, hdfs, mr, yarn"
+    exit 1
+    ;;
+
+esac

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java

@@ -124,6 +124,11 @@ public abstract class TestConfigurationFieldsBase {
    */
   private Map<String,String> configurationMemberVariables = null;
 
+  /**
+   * Member variable to store Configuration variables for later reference.
+   */
+  private Map<String,String> configurationDefaultVariables = null;
+
   /**
    * Member variable to store XML properties for later comparison.
    */
@@ -146,6 +151,7 @@ public abstract class TestConfigurationFieldsBase {
    */
   protected boolean configDebug = false;
   protected boolean xmlDebug = false;
+  protected boolean defaultDebug = false;
 
   /**
    * Abstract method to be used by subclasses for initializing base
@@ -316,6 +322,79 @@ public abstract class TestConfigurationFieldsBase {
     return retVal;
   }
 
+  /**
+   * Utility function to extract &quot;public static final&quot; default
+   * member variables from a Configuration type class.
+   *
+   * @param fields The class member variables
+   * @return HashMap containing <DefaultVariableName,DefaultValue> entries
+   */
+  private HashMap<String,String>
+      extractDefaultVariablesFromConfigurationFields(Field[] fields) {
+    // Sanity Check
+    if (fields==null) {
+      return null;
+    }
+
+    HashMap<String,String> retVal = new HashMap<String,String>();
+
+    // Setup regexp for valid properties
+    String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$";
+    Pattern p = Pattern.compile(propRegex);
+
+    // Iterate through class member variables
+    int totalFields = 0;
+    String value;
+    for (Field f : fields) {
+      // Filter out anything that isn't "public static final"
+      if (!Modifier.isStatic(f.getModifiers()) ||
+          !Modifier.isPublic(f.getModifiers()) ||
+          !Modifier.isFinal(f.getModifiers())) {
+        continue;
+      }
+      // Special: Stuff any property beginning with "DEFAULT_" into a
+      // different hash for later processing
+      if (f.getName().startsWith("DEFAULT_") ||
+          f.getName().endsWith("_DEFAULT")) {
+        if (retVal.containsKey(f.getName())) {
+          continue;
+        }
+        try {
+          if (f.getType().getName().equals("java.lang.String")) {
+            String sValue = (String) f.get(null);
+            retVal.put(f.getName(),sValue);
+          } else if (f.getType().getName().equals("short")) {
+            short shValue = (short) f.get(null);
+            retVal.put(f.getName(),Integer.toString(shValue));
+          } else if (f.getType().getName().equals("int")) {
+            int iValue = (int) f.get(null);
+            retVal.put(f.getName(),Integer.toString(iValue));
+          } else if (f.getType().getName().equals("long")) {
+            long lValue = (long) f.get(null);
+            retVal.put(f.getName(),Long.toString(lValue));
+          } else if (f.getType().getName().equals("float")) {
+            float fValue = (float) f.get(null);
+            retVal.put(f.getName(),Float.toString(fValue));
+          } else if (f.getType().getName().equals("double")) {
+            double dValue = (double) f.get(null);
+            retVal.put(f.getName(),Double.toString(dValue));
+          } else if (f.getType().getName().equals("boolean")) {
+            boolean bValue = (boolean) f.get(null);
+            retVal.put(f.getName(),Boolean.toString(bValue));
+          } else {
+            if (defaultDebug) {
+              System.out.println("Config variable " + f.getName() + " has unknown type " + f.getType().getName());
+            }
+          }
+        } catch (IllegalAccessException iaException) {
+          iaException.printStackTrace();
+        }
+      }
+    }
+
+    return retVal;
+  }
+
   /**
    * Perform set difference operation on keyMap2 from keyMap1.
    *
@@ -374,6 +453,26 @@ public abstract class TestConfigurationFieldsBase {
       System.out.println("");
     }
 
+    // Create default configuration variable key/value map
+    if (defaultDebug) {
+      System.out.println("Reading Config property files for defaults");
+      System.out.println("");
+    }
+    configurationDefaultVariables = new HashMap<String,String>();
+    for (Class c : configurationClasses) {
+      Field[] fields = c.getDeclaredFields();
+      Map<String,String> defaultMap =
+          extractDefaultVariablesFromConfigurationFields(fields);
+      if (defaultMap!=null) {
+        configurationDefaultVariables.putAll(defaultMap);
+      }
+    }
+    if (defaultDebug) {
+      System.out.println("");
+      System.out.println("=====");
+      System.out.println("");
+    }
+
     // Find class members not in the XML file
     configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields
         (configurationMemberVariables, xmlKeyValueMap);
@@ -464,4 +563,160 @@ public abstract class TestConfigurationFieldsBase {
       assertTrue(configErrorMsg.toString(), missingConfigSize==0);
     }
   }
+
+  /**
+   * For each property in the XML file, verify that the value matches
+   * up to the default if one exists.
+   */
+  @Test
+  public void testXmlAgainstDefaultValuesInConfigurationClass() {
+    // Error if subclass hasn't set class members
+    assertTrue(xmlFilename!=null);
+    assertTrue(configurationMemberVariables!=null);
+    assertTrue(configurationDefaultVariables!=null);
+
+    HashSet<String> xmlPropertiesWithEmptyValue = new HashSet<String>();
+    HashSet<String> configPropertiesWithNoDefaultConfig = new HashSet<String>();
+    HashMap<String,String> xmlPropertiesMatchingConfigDefault =
+        new HashMap<String,String>();
+    // Ugly solution.  Should have tuple-based solution.
+    HashMap<HashMap<String,String>,HashMap<String,String>> mismatchingXmlConfig =
+        new HashMap<HashMap<String,String>,HashMap<String,String>>();
+
+    for (Map.Entry<String,String> xEntry : xmlKeyValueMap.entrySet()) {
+      String xmlProperty = xEntry.getKey();
+      String xmlDefaultValue = xEntry.getValue();
+      String configProperty = configurationMemberVariables.get(xmlProperty);
+      if (configProperty!=null) {
+        String defaultConfigName = null;
+        String defaultConfigValue = null;
+
+        // Type 1: Prepend DEFAULT_
+        String defaultNameCheck1 = "DEFAULT_" + configProperty;
+        String defaultValueCheck1 = configurationDefaultVariables
+            .get(defaultNameCheck1);
+        // Type 2: Swap _KEY suffix with _DEFAULT suffix
+        String defaultNameCheck2 = null;
+        if (configProperty.endsWith("_KEY")) {
+          defaultNameCheck2 = configProperty
+              .substring(0,configProperty.length()-4) + "_DEFAULT";
+        }
+        String defaultValueCheck2 = configurationDefaultVariables
+            .get(defaultNameCheck2);
+        // Type Last: Append _DEFAULT suffix
+        String defaultNameCheck3 = configProperty + "_DEFAULT";
+        String defaultValueCheck3 = configurationDefaultVariables
+            .get(defaultNameCheck3);
+
+        // Pick the default value that exists
+        if (defaultValueCheck1!=null) {
+          defaultConfigName = defaultNameCheck1;
+          defaultConfigValue = defaultValueCheck1;
+        } else if (defaultValueCheck2!=null) {
+          defaultConfigName = defaultNameCheck2;
+          defaultConfigValue = defaultValueCheck2;
+        } else if (defaultValueCheck3!=null) {
+          defaultConfigName = defaultNameCheck3;
+          defaultConfigValue = defaultValueCheck3;
+        }
+
+        if (defaultConfigValue!=null) {
+          if (xmlDefaultValue==null) {
+            xmlPropertiesWithEmptyValue.add(xmlProperty);
+          } else if (!xmlDefaultValue.equals(defaultConfigValue)) {
+            HashMap<String,String> xmlEntry =
+                new HashMap<String,String>();
+            xmlEntry.put(xmlProperty,xmlDefaultValue);
+            HashMap<String,String> configEntry =
+                new HashMap<String,String>();
+            configEntry.put(defaultConfigName,defaultConfigValue);
+            mismatchingXmlConfig.put(xmlEntry,configEntry);
+           } else {
+            xmlPropertiesMatchingConfigDefault
+                .put(xmlProperty, defaultConfigName);
+          }
+        } else {
+          configPropertiesWithNoDefaultConfig.add(configProperty);
+        }
+      } else {
+      }
+    }
+
+    // Print out any unknown mismatching XML value/Config default value
+    System.out.println(this.xmlFilename + " has " +
+        mismatchingXmlConfig.size() +
+        " properties that do not match the default Config value");
+    if (mismatchingXmlConfig.size()==0) {
+      System.out.println("  (None)");
+    } else {
+       for (Map.Entry<HashMap<String,String>,HashMap<String,String>> xcEntry :
+           mismatchingXmlConfig.entrySet()) {
+         HashMap<String,String> xmlMap = xcEntry.getKey();
+         HashMap<String,String> configMap = xcEntry.getValue();
+         for (Map.Entry<String,String> xmlEntry : xmlMap.entrySet()) {
+           System.out.println("  XML Property: " + xmlEntry.getKey());
+           System.out.println("  XML Value:    " + xmlEntry.getValue());
+         }
+         for (Map.Entry<String,String> configEntry : configMap.entrySet()) {
+           System.out.println("  Config Name:  " + configEntry.getKey());
+           System.out.println("  Config Value: " + configEntry.getValue());
+         }
+         System.out.println("");
+       }
+    }
+    System.out.println();
+
+    // Print out Config properties that have no corresponding DEFAULT_*
+    // variable and cannot do any XML comparison (i.e. probably needs to
+    // be checked by hand)
+    System.out.println("Configuration(s) have " +
+        configPropertiesWithNoDefaultConfig.size() +
+        " properties with no corresponding default member variable.  These" +
+        " will need to be verified manually.");
+    if (configPropertiesWithNoDefaultConfig.size()==0) {
+      System.out.println("  (None)");
+    } else {
+      Iterator<String> cItr = configPropertiesWithNoDefaultConfig.iterator();
+      while (cItr.hasNext()) {
+        System.out.println("  " + cItr.next());
+      }
+    }
+    System.out.println();
+
+    // MAYBE TODO Print out any known mismatching XML value/Config default
+
+    // Print out XML properties that have empty values (i.e. should result
+    // in code-based default)
+    System.out.println(this.xmlFilename + " has " +
+        xmlPropertiesWithEmptyValue.size() + " properties with empty values");
+    if (xmlPropertiesWithEmptyValue.size()==0) {
+      System.out.println("  (None)");
+    } else {
+      Iterator<String> xItr = xmlPropertiesWithEmptyValue.iterator();
+      while (xItr.hasNext()) {
+        System.out.println("  " + xItr.next());
+      }
+    }
+    System.out.println();
+
+    // Print out any matching XML value/Config default value
+    System.out.println(this.xmlFilename + " has " +
+        xmlPropertiesMatchingConfigDefault.size() +
+        " properties which match a corresponding Config variable");
+    if (xmlPropertiesMatchingConfigDefault.size()==0) {
+      System.out.println("  (None)");
+    } else {
+      for (Map.Entry<String,String> xcEntry :
+          xmlPropertiesMatchingConfigDefault.entrySet()) {
+        System.out.println("  " + xcEntry.getKey() + " / " +
+            xcEntry.getValue());
+      }
+    }
+    System.out.println();
+
+    // Test separator
+    System.out.println();
+    System.out.println("=====");
+    System.out.println();
+  }
 }