HADOOP-463. Add variable expansion to config files. Contributed by Michel.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@437873 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -89,6 +89,15 @@ Trunk (unreleased changes)
 22. HADOOP-437.  contrib/streaming: Add support for gzipped inputs.
     (Michel Tourn via cutting)
 
+32. HADOOP-463.  Add variable expansion to config files.
+    Configuration property values may now contain variable
+    expressions.  A variable is referenced with the syntax
+    '${variable}'.  Variables values are found first in the
+    configuration, and then in Java system properties.  The default
+    configuration is modified so that temporary directories are now
+    under ${hadoop.tmp.dir}, which is, by default,
+    /tmp/hadoop-${user.name}.  (Michel Tourn via cutting)
+
 
 Release 0.5.0 - 2006-08-04
 

conf/hadoop-default.xml

@@ -7,6 +7,15 @@
 
 <configuration>
 
+<!--- global properties -->
+
+<property>
+  <name>hadoop.tmp.dir</name>
+  <value>/tmp/hadoop-${user.name}</value>
+  <description>A base for other temporary directories.</description>
+</property>
+
+
 <!--- logging properties -->
 
 <property>
@@ -125,14 +134,14 @@ creations/deletions), or "all".</description>
 
 <property>
   <name>dfs.name.dir</name>
-  <value>/tmp/hadoop/dfs/name</value>
+  <value>${hadoop.tmp.dir}/dfs/name</value>
   <description>Determines where on the local filesystem the DFS name node
       should store the name table.</description>
 </property>
 
 <property>
   <name>dfs.data.dir</name>
-  <value>/tmp/hadoop/dfs/data</value>
+  <value>${hadoop.tmp.dir}/dfs/data</value>
   <description>Determines where on the local filesystem an DFS data node
   should store its blocks.  If this is a comma-delimited
   list of directories, then data will be stored in all named
@@ -221,7 +230,7 @@ creations/deletions), or "all".</description>
 
 <property>
   <name>mapred.local.dir</name>
-  <value>/tmp/hadoop/mapred/local</value>
+  <value>${hadoop.tmp.dir}/mapred/local</value>
   <description>The local directory where MapReduce stores intermediate
   data files.  May be a comma-separated list of
   directories on different devices in order to spread disk i/o.
@@ -231,14 +240,14 @@ creations/deletions), or "all".</description>
 
 <property>
   <name>mapred.system.dir</name>
-  <value>/tmp/hadoop/mapred/system</value>
+  <value>${hadoop.tmp.dir}/mapred/system</value>
   <description>The shared directory where MapReduce stores control files.
   </description>
 </property>
 
 <property>
   <name>mapred.temp.dir</name>
-  <value>/tmp/hadoop/mapred/temp</value>
+  <value>${hadoop.tmp.dir}/mapred/temp</value>
   <description>A shared directory for temporary files.
   </description>
 </property>

src/java/org/apache/hadoop/conf/Configuration.java

@@ -17,6 +17,8 @@
 package org.apache.hadoop.conf;
 
 import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.net.URL;
 import java.io.*;
 
@@ -37,7 +39,7 @@ import org.apache.hadoop.fs.Path;
 /** Provides access to configuration parameters.  Configurations are specified
  * by resources.  A resource contains a set of name/value pairs.
  *
- * <p>Each resources is named by either a String or by a Path.  If named by a
+ * <p>Each resource is named by either a String or by a Path.  If named by a
  * String, then the classpath is examined for a file with that name.  If a
  * File, then the local filesystem is examined directly, without referring to
  * the CLASSPATH.
@@ -49,6 +51,21 @@ import org.apache.hadoop.fs.Path;
  * <p>Hadoop's default resource is the String "hadoop-default.xml" and its
  * final resource is the String "hadoop-site.xml".  Other tools built on Hadoop
  * may specify additional resources.
+ * 
+ * <p>The values returned by most <tt>get*</tt> methods are based on String representations. 
+ * This String is processed for <b>variable expansion</b>. The available variables are the 
+ * <em>System properties</em> and the <em>other properties</em> defined in this Configuration.
+ * <p>The only <tt>get*</tt> method that is not processed for variable expansion is
+ * {@link getObject} (as it cannot assume that the returned values are String). 
+ * You can use <tt>getObject</tt> to obtain the raw value of a String property without 
+ * variable expansion: if <tt>(String)conf.getObject("my.jdk")</tt> is <tt>"JDK ${java.version}"</tt>
+ * then conf.get("my.jdk")</tt> is <tt>"JDK 1.5.0"</tt> 
+ * <p> Example XML config using variables:<br><tt>
+ * &lt;name>basedir&lt;/name>&lt;value>/user/${user.name}&lt;/value><br> 
+ * &lt;name>tempdir&lt;/name>&lt;value>${basedir}/tmp&lt;/value><br>
+ * </tt>When conf.get("tempdir") is called:<br>
+ * <tt>${basedir}</tt> is resolved to another property in this Configuration.
+ * Then <tt>${user.name}</tt> is resolved to a System property.
  */
 public class Configuration {
   private static final Log LOG =
@@ -137,9 +154,41 @@ public class Configuration {
     else return defaultValue;
   }
   
+  private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
+  private static int MAX_SUBST = 20;
+
+  private String substituteVars(String expr) {
+    if(expr == null) {
+      return null;
+    }
+    Matcher match = varPat.matcher("");
+    String eval = expr;
+    for(int s=0; s<MAX_SUBST; s++) {
+      match.reset(eval);
+      if(! match.find()) {
+        return eval;
+      }
+      String var = match.group();
+      var = var.substring(2, var.length()-1); // remove ${ .. }
+      String val = System.getProperty(var);
+      if(val == null) {
+        val = (String)this.getObject(var);
+      }
+      if(val == null) {
+        return eval; // return literal ${var}: var is unbound
+      }
+      // substitute
+      eval = eval.substring(0, match.start())+val+eval.substring(match.end());
+    }
+    throw new IllegalStateException("Variable substitution depth too large: " 
+                                    + MAX_SUBST + " " + expr);
+  }
+  
   /** Returns the value of the <code>name</code> property, or null if no
    * such property exists. */
-  public String get(String name) { return getProps().getProperty(name);}
+  public String get(String name) {
+    return substituteVars(getProps().getProperty(name));
+  }
 
   /** Sets the value of the <code>name</code> property. */
   public void set(String name, Object value) {
@@ -150,9 +199,9 @@ public class Configuration {
    * exists, then <code>defaultValue</code> is returned.
    */
   public String get(String name, String defaultValue) {
-     return getProps().getProperty(name, defaultValue);
+     return substituteVars(getProps().getProperty(name, defaultValue));
   }
-  
+    
   /** Returns the value of the <code>name</code> property as an integer.  If no
    * such property is specified, or if the specified value is not a valid
    * integer, then <code>defaultValue</code> is returned.

src/test/org/apache/hadoop/conf/TestConfiguration.java

@@ -0,0 +1,119 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.conf;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.apache.hadoop.fs.Path;
+
+import junit.framework.TestCase;
+
+
+public class TestConfiguration extends TestCase {
+    
+  public void testVariableSubstitution() throws IOException {
+    Configuration conf = new Configuration();
+    final String CONFIG = new File("./test-config.xml").getAbsolutePath();
+    this.out = new BufferedWriter(new FileWriter(CONFIG));
+    out.write("<?xml version=\"1.0\"?>\n");
+    out.write("<configuration>\n");
+    declareProperty("my.int", "${intvar}", "42");
+    declareProperty("intvar", "42", "42");
+    declareProperty("my.base", "/tmp/${user.name}", UNSPEC);
+    declareProperty("my.file", "hello", "hello");
+    declareProperty("my.suffix", ".txt", ".txt");
+    declareProperty("my.relfile", "${my.file}${my.suffix}", "hello.txt");
+    declareProperty("my.fullfile", "${my.base}/${my.file}${my.suffix}", UNSPEC);
+    // check that undefined variables are returned as-is
+    declareProperty("my.failsexpand", "a${my.undefvar}b", "a${my.undefvar}b");
+    out.write("</configuration>\n");
+    out.close();
+    Path fileResource = new Path(CONFIG);
+    conf.addDefaultResource(fileResource);
+
+    Iterator it = props.iterator();
+    while(it.hasNext()) {
+      Prop p = (Prop)it.next();
+      System.out.println("p=" + p.name);
+      String gotVal = conf.get(p.name);
+      String gotRawVal = (String)conf.getObject(p.name);
+      assertEq(p.val, gotRawVal);
+      if(p.expectEval == UNSPEC) {
+        // expansion is system-dependent (uses System properties)
+        // can't do exact match so just check that all variables got expanded
+        assertTrue(gotVal != null && -1 == gotVal.indexOf("${"));
+      } else {
+        assertEq(p.expectEval, gotVal);
+      }
+    }
+      
+    // check that expansion also occurs for getInt()
+    assertTrue(conf.getInt("intvar", -1) == 42);
+    assertTrue(conf.getInt("my.int", -1) == 42);
+      
+    new File(CONFIG).delete();
+  }
+    
+  public static void assertEq(Object a, Object b) {
+    System.out.println("assertEq: " + a + ", " + b);
+    assertEquals(a, b);
+  }
+
+  static class Prop {
+    String name;
+    String val;
+    String expectEval;
+  }
+
+  final String UNSPEC = null;
+  ArrayList props = new ArrayList();
+
+  void declareProperty(String name, String val, String expectEval)
+    throws IOException {
+    appendProperty(name, val);
+    Prop p = new Prop();
+    p.name = name;
+    p.val = val;
+    p.expectEval = expectEval;
+    props.add(p);
+  }
+
+  void appendProperty(String name, String val) throws IOException {
+    out.write("<property>");
+    out.write("<name>");
+    out.write(name);
+    out.write("</name>");
+    out.write("<value>");
+    out.write(val);
+    out.write("</value>");
+    out.write("</property>\n");
+  }
+
+  BufferedWriter out;
+	
+  public static void main(String[] argv) throws Exception {
+    junit.textui.TestRunner.main(new String[]{
+      TestConfiguration.class.getName()
+    });
+  }
+}