HADOOP-4944. A configuration file can include other configuration

files. (Rama Ramasamy via dhruba)



git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@739416 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -402,6 +402,9 @@ Release 0.20.0 - Unreleased
     HADOOP-4939. Adds a test that would inject random failures for tasks in 
     large jobs and would also inject TaskTracker failures. (ddas)
 
+    HADOOP-4944. A configuration file can include other configuration
+    files. (Rama Ramasamy via dhruba)
+
   OPTIMIZATIONS
 
     HADOOP-3293. Fixes FileInputFormat to do provide locations for splits

src/core/org/apache/hadoop/conf/Configuration.java

@@ -1046,9 +1046,13 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
         = DocumentBuilderFactory.newInstance();
       //ignore all comments inside the xml file
       docBuilderFactory.setIgnoringComments(true);
+
+      //allow includes in the xml file
+      docBuilderFactory.setNamespaceAware(true);
+      docBuilderFactory.setXIncludeAware(true);
       DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
       Document doc = null;
-
+      Element root = null;
 
       if (name instanceof URL) {                  // an URL resource
         URL url = (URL)name;
@@ -1088,15 +1092,19 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
         } finally {
           ((InputStream)name).close();
         }
+      } else if (name instanceof Element) {
+        root = (Element)name;
       }
 
-      if (doc == null) {
+      if (doc == null && root == null) {
         if (quiet)
           return;
         throw new RuntimeException(name + " not found");
       }
 
-      Element root = doc.getDocumentElement();
+      if (root == null) {
+        root = doc.getDocumentElement();
+      }
       if (!"configuration".equals(root.getTagName()))
         LOG.fatal("bad conf file: top-level element not <configuration>");
       NodeList props = root.getChildNodes();
@@ -1105,6 +1113,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
         if (!(propNode instanceof Element))
           continue;
         Element prop = (Element)propNode;
+        if ("configuration".equals(prop.getTagName())) {
+          loadResource(properties, prop, quiet);
+          continue;
+        }
         if (!"property".equals(prop.getTagName()))
           LOG.warn("bad conf file: element not <property>");
         NodeList fields = prop.getChildNodes();

src/test/org/apache/hadoop/conf/TestConfiguration.java

@@ -62,7 +62,11 @@ public class TestConfiguration extends TestCase {
     out.write("</configuration>\n");
     out.close();
   }
-  
+
+  private void addInclude(String filename) throws IOException{
+    out.write("<xi:include href=\"" + filename + "\" xmlns:xi=\"http://www.w3.org/2001/XInclude\"  />\n ");
+  }
+
   public void testVariableSubstitution() throws IOException {
     out=new BufferedWriter(new FileWriter(CONFIG));
     startConfig();
@@ -228,6 +232,32 @@ public class TestConfiguration extends TestCase {
       fileResource.toString();
     assertEquals(expectedOutput, conf.toString());
   }
+  
+  public void testIncludes() throws Exception {
+    tearDown();
+    System.out.println("XXX testIncludes");
+    out=new BufferedWriter(new FileWriter(CONFIG2));
+    startConfig();
+    appendProperty("a","b");
+    appendProperty("c","d");
+    endConfig();
+
+    out=new BufferedWriter(new FileWriter(CONFIG));
+    startConfig();
+    addInclude(CONFIG2);
+    appendProperty("e","f");
+    appendProperty("g","h");
+    endConfig();
+
+    // verify that the includes file contains all properties
+    Path fileResource = new Path(CONFIG);
+    conf.addResource(fileResource);
+    assertEquals(conf.get("a"), "b"); 
+    assertEquals(conf.get("c"), "d"); 
+    assertEquals(conf.get("e"), "f"); 
+    assertEquals(conf.get("g"), "h"); 
+    tearDown();
+  }
 
   BufferedWriter out;