HADOOP-4631. Splits the configuration into three parts - one for core,

one for mapred and the last one for HDFS. Contributed by Sharad Agarwal.


git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@726883 13f79535-47bb-0310-9956-ffa450edef68

.gitignore

@@ -23,6 +23,9 @@ conf/masters
 conf/slaves
 conf/hadoop-env.sh
 conf/hadoop-site.xml
+conf/core-site.xml
+conf/mapred-site.xml
+conf/hdfs-site.xml
 conf/hadoop-policy.xml
 conf/capacity-scheduler.xml
 docs/api/

CHANGES.txt

@@ -54,6 +54,9 @@ Trunk (unreleased changes)
     percentage in capacity scheduler UI. (Sreekanth Ramakrishnan via
     yhemanth)
 
+    HADOOP-4631. Splits the configuration into three parts - one for core,
+    one for mapred and the last one for HDFS. (Sharad Agarwal via cdouglas)
+
   NEW FEATURES
 
     HADOOP-4575. Add a proxy service for relaying HsftpFileSystem requests.

build.xml

@@ -342,6 +342,7 @@
 
     <copy todir="${build.classes}">
       <fileset dir="${core.src.dir}" includes="**/*.properties"/>
+      <fileset dir="${core.src.dir}" includes="core-default.xml"/>
     </copy>
      
   </target>
@@ -378,6 +379,7 @@
     
     <copy todir="${build.classes}">
       <fileset dir="${mapred.src.dir}" includes="**/*.properties"/>
+      <fileset dir="${mapred.src.dir}" includes="mapred-default.xml"/>
     </copy>
   </target>
 
@@ -413,6 +415,7 @@
 
     <copy todir="${build.classes}">
      <fileset dir="${hdfs.src.dir}" includes="**/*.properties"/>
+     <fileset dir="${hdfs.src.dir}" includes="hdfs-default.xml"/>
     </copy>
   </target>
 
@@ -547,7 +550,6 @@
           <attribute name="Implementation-Vendor" value="Apache"/>
         </section>
       </manifest>
-      <fileset file="${conf.dir}/hadoop-default.xml"/>
       <fileset file="${conf.dir}/commons-logging.properties"/>
       <fileset file="${conf.dir}/log4j.properties"/>
       <fileset file="${conf.dir}/hadoop-metrics.properties"/>
@@ -860,8 +862,12 @@
     <copy todir="${docs.dir}">
       <fileset dir="${docs.src}/build/site/" />
     </copy>
-    <style basedir="${conf.dir}" destdir="${docs.dir}"
-           includes="hadoop-default.xml" style="conf/configuration.xsl"/>
+    <style basedir="${core.src.dir}" destdir="${docs.dir}"
+           includes="core-default.xml" style="conf/configuration.xsl"/>
+    <style basedir="${hdfs.src.dir}" destdir="${docs.dir}"
+           includes="hdfs-default.xml" style="conf/configuration.xsl"/>
+    <style basedir="${mapred.src.dir}" destdir="${docs.dir}"
+           includes="mapred-default.xml" style="conf/configuration.xsl"/>
     <antcall target="changes-to-html"/>
     <antcall target="cn-docs"/>
   </target>
@@ -876,8 +882,12 @@
     <copy todir="${cndocs.dir}">
       <fileset dir="${cndocs.src}/build/site/" />
     </copy>
-    <style basedir="${conf.dir}" destdir="${cndocs.dir}"
-          includes="hadoop-default.xml" style="conf/configuration.xsl"/>
+    <style basedir="${core.src.dir}" destdir="${cndocs.dir}"
+           includes="core-default.xml" style="conf/configuration.xsl"/>
+    <style basedir="${hdfs.src.dir}" destdir="${cndocs.dir}"
+           includes="hdfs-default.xml" style="conf/configuration.xsl"/>
+    <style basedir="${mapred.src.dir}" destdir="${cndocs.dir}"
+           includes="mapred-default.xml" style="conf/configuration.xsl"/>
     <antcall target="changes-to-html"/>
   </target>
 

conf/hdfs-site.xml.template

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+</configuration>

conf/mapred-site.xml.template

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+</configuration>

src/c++/libhdfs/tests/conf/core-site.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Values used when running libhdfs unit tests. -->
+<!-- This is mostly empty, to use the default values, overriding the -->
+<!-- potentially user-editted core-site.xml in the conf/ directory.  -->
+
+<configuration>
+
+<property>
+  <name>hadoop.tmp.dir</name>
+  <value>build/test/libhdfs</value>
+  <description>A base for other temporary directories.</description>
+</property>
+
+
+<property>
+  <name>fs.default.name</name>
+  <value>localhost:23000</value>
+  <description>The name of the default file system.  Either the
+  literal string "local" or a host:port for DFS.</description>
+</property>
+
+</configuration>

src/c++/libhdfs/tests/conf/hadoop-site.xml

@@ -7,27 +7,5 @@
 
 <configuration>
 
-<property>
-  <name>hadoop.tmp.dir</name>
-  <value>build/test/libhdfs</value>
-  <description>A base for other temporary directories.</description>
-</property>
-
-
-<property>
-  <name>fs.default.name</name>
-  <value>localhost:23000</value>
-  <description>The name of the default file system.  Either the
-  literal string "local" or a host:port for DFS.</description>
-</property>
-
-<property>
-  <name>dfs.replication</name>
-  <value>1</value>
-  <description>Default block replication.
-  The actual number of replications can be specified when the file is created.
-  The default is used if replication is not specified in create time.
-  </description>
-</property>
 
 </configuration>

src/c++/libhdfs/tests/conf/hdfs-site.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+<property>
+  <name>dfs.replication</name>
+  <value>1</value>
+  <description>Default block replication.
+  The actual number of replications can be specified when the file is created.
+  The default is used if replication is not specified in create time.
+  </description>
+</property>
+
+</configuration>

src/c++/libhdfs/tests/conf/mapred-site.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+</configuration>

src/contrib/test/core-site.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Values used when running unit tests.  This is mostly empty, to -->
+<!-- use of the default values, overriding the potentially -->
+<!-- user-editted core-site.xml in the conf/ directory.  -->
+
+<configuration>
+
+<property>
+  <name>hadoop.tmp.dir</name>
+  <value>${build.test}</value>
+  <description>A base for other temporary directories.</description>
+</property>
+
+</configuration>

src/contrib/test/hadoop-site.xml

@@ -8,16 +8,5 @@
 <configuration>
 
 
-<property>
-  <name>hadoop.tmp.dir</name>
-  <value>${build.test}</value>
-  <description>A base for other temporary directories.</description>
-</property>
-
-<property>
-  <name>mapred.system.dir</name>
-  <value>build/contrib/${contrib.name}/test/system</value>
-</property>
-
 
 </configuration>

src/contrib/test/hdfs-site.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+
+</configuration>

src/contrib/test/mapred-site.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+<property>
+  <name>mapred.system.dir</name>
+  <value>build/contrib/${contrib.name}/test/system</value>
+</property>
+
+</configuration>

src/core/core-default.xml

@@ -0,0 +1,434 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Do not modify this file directly.  Instead, copy entries that you -->
+<!-- wish to modify from this file into core-site.xml and change them -->
+<!-- there.  If core-site.xml does not already exist, create it.      -->
+
+<configuration>
+
+<!--- global properties -->
+
+<property>
+  <name>hadoop.tmp.dir</name>
+  <value>/tmp/hadoop-${user.name}</value>
+  <description>A base for other temporary directories.</description>
+</property>
+
+<property>
+  <name>hadoop.native.lib</name>
+  <value>true</value>
+  <description>Should native hadoop libraries, if present, be used.</description>
+</property>
+
+<property>
+  <name>hadoop.http.filter.initializers</name>
+  <value></value>
+  <description>A comma separated list of class names. Each class in the list 
+  must extend org.apache.hadoop.http.FilterInitializer. The corresponding 
+  Filter will be initialized. Then, the Filter will be applied to all user 
+  facing jsp and servlet web pages.  The ordering of the list defines the 
+  ordering of the filters.</description>
+</property>
+
+<property>
+  <name>hadoop.security.authorization</name>
+  <value>false</value>
+  <description>Is service-level authorization enabled?</description>
+</property>
+
+<!--- logging properties -->
+
+<property>
+  <name>hadoop.logfile.size</name>
+  <value>10000000</value>
+  <description>The max size of each log file</description>
+</property>
+
+<property>
+  <name>hadoop.logfile.count</name>
+  <value>10</value>
+  <description>The max number of log files</description>
+</property>
+
+<!-- i/o properties -->
+<property>
+  <name>io.file.buffer.size</name>
+  <value>4096</value>
+  <description>The size of buffer for use in sequence files.
+  The size of this buffer should probably be a multiple of hardware
+  page size (4096 on Intel x86), and it determines how much data is
+  buffered during read and write operations.</description>
+</property>
+  
+<property>
+  <name>io.bytes.per.checksum</name>
+  <value>512</value>
+  <description>The number of bytes per checksum.  Must not be larger than
+  io.file.buffer.size.</description>
+</property>
+
+<property>
+  <name>io.skip.checksum.errors</name>
+  <value>false</value>
+  <description>If true, when a checksum error is encountered while
+  reading a sequence file, entries are skipped, instead of throwing an
+  exception.</description>
+</property>
+
+<property>
+  <name>io.compression.codecs</name>
+  <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec</value>
+  <description>A list of the compression codec classes that can be used 
+               for compression/decompression.</description>
+</property>
+
+<property>
+  <name>io.serializations</name>
+  <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
+  <description>A list of serialization classes that can be used for
+  obtaining serializers and deserializers.</description>
+</property>
+
+<!-- file system properties -->
+
+<property>
+  <name>fs.default.name</name>
+  <value>file:///</value>
+  <description>The name of the default file system.  A URI whose
+  scheme and authority determine the FileSystem implementation.  The
+  uri's scheme determines the config property (fs.SCHEME.impl) naming
+  the FileSystem implementation class.  The uri's authority is used to
+  determine the host, port, etc. for a filesystem.</description>
+</property>
+
+<property>
+  <name>fs.trash.interval</name>
+  <value>0</value>
+  <description>Number of minutes between trash checkpoints.
+  If zero, the trash feature is disabled.
+  </description>
+</property>
+
+<property>
+  <name>fs.file.impl</name>
+  <value>org.apache.hadoop.fs.LocalFileSystem</value>
+  <description>The FileSystem for file: uris.</description>
+</property>
+
+<property>
+  <name>fs.hdfs.impl</name>
+  <value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
+  <description>The FileSystem for hdfs: uris.</description>
+</property>
+
+<property>
+  <name>fs.s3.impl</name>
+  <value>org.apache.hadoop.fs.s3.S3FileSystem</value>
+  <description>The FileSystem for s3: uris.</description>
+</property>
+
+<property>
+  <name>fs.s3n.impl</name>
+  <value>org.apache.hadoop.fs.s3native.NativeS3FileSystem</value>
+  <description>The FileSystem for s3n: (Native S3) uris.</description>
+</property>
+
+<property>
+  <name>fs.kfs.impl</name>
+  <value>org.apache.hadoop.fs.kfs.KosmosFileSystem</value>
+  <description>The FileSystem for kfs: uris.</description>
+</property>
+
+<property>
+  <name>fs.hftp.impl</name>
+  <value>org.apache.hadoop.hdfs.HftpFileSystem</value>
+</property>
+
+<property>
+  <name>fs.hsftp.impl</name>
+  <value>org.apache.hadoop.hdfs.HsftpFileSystem</value>
+</property>
+
+<property>
+  <name>fs.ftp.impl</name>
+  <value>org.apache.hadoop.fs.ftp.FTPFileSystem</value>
+  <description>The FileSystem for ftp: uris.</description>
+</property>
+
+<property>
+  <name>fs.ramfs.impl</name>
+  <value>org.apache.hadoop.fs.InMemoryFileSystem</value>
+  <description>The FileSystem for ramfs: uris.</description>
+</property>
+
+<property>
+  <name>fs.har.impl</name>
+  <value>org.apache.hadoop.fs.HarFileSystem</value>
+  <description>The filesystem for Hadoop archives. </description>
+</property>
+
+<property>
+  <name>fs.checkpoint.dir</name>
+  <value>${hadoop.tmp.dir}/dfs/namesecondary</value>
+  <description>Determines where on the local filesystem the DFS secondary
+      name node should store the temporary images to merge.
+      If this is a comma-delimited list of directories then the image is
+      replicated in all of the directories for redundancy.
+  </description>
+</property>
+
+<property>
+  <name>fs.checkpoint.edits.dir</name>
+  <value>${fs.checkpoint.dir}</value>
+  <description>Determines where on the local filesystem the DFS secondary
+      name node should store the temporary edits to merge.
+      If this is a comma-delimited list of directoires then teh edits is
+      replicated in all of the directoires for redundancy.
+      Default value is same as fs.checkpoint.dir
+  </description>
+</property>
+
+<property>
+  <name>fs.checkpoint.period</name>
+  <value>3600</value>
+  <description>The number of seconds between two periodic checkpoints.
+  </description>
+</property>
+
+<property>
+  <name>fs.checkpoint.size</name>
+  <value>67108864</value>
+  <description>The size of the current edit log (in bytes) that triggers
+       a periodic checkpoint even if the fs.checkpoint.period hasn't expired.
+  </description>
+</property>
+
+
+
+<property>
+  <name>fs.s3.block.size</name>
+  <value>67108864</value>
+  <description>Block size to use when writing files to S3.</description>
+</property>
+
+<property>
+  <name>fs.s3.buffer.dir</name>
+  <value>${hadoop.tmp.dir}/s3</value>
+  <description>Determines where on the local filesystem the S3 filesystem
+  should store files before sending them to S3
+  (or after retrieving them from S3).
+  </description>
+</property>
+
+<property>
+  <name>fs.s3.maxRetries</name>
+  <value>4</value>
+  <description>The maximum number of retries for reading or writing files to S3, 
+  before we signal failure to the application.
+  </description>
+</property>
+
+<property>
+  <name>fs.s3.sleepTimeSeconds</name>
+  <value>10</value>
+  <description>The number of seconds to sleep between each S3 retry.
+  </description>
+</property>
+
+
+<property>
+  <name>local.cache.size</name>
+  <value>10737418240</value>
+  <description>The limit on the size of cache you want to keep, set by default
+  to 10GB. This will act as a soft limit on the cache directory for out of band data.
+  </description>
+</property>
+            
+<property>
+  <name>io.seqfile.compress.blocksize</name>
+  <value>1000000</value>
+  <description>The minimum block size for compression in block compressed 
+          SequenceFiles.
+  </description>
+</property>
+
+<property>
+  <name>io.seqfile.lazydecompress</name>
+  <value>true</value>
+  <description>Should values of block-compressed SequenceFiles be decompressed
+          only when necessary.
+  </description>
+</property>
+
+<property>
+  <name>io.seqfile.sorter.recordlimit</name>
+  <value>1000000</value>
+  <description>The limit on number of records to be kept in memory in a spill 
+          in SequenceFiles.Sorter
+  </description>
+</property>
+
+ <property>
+  <name>io.mapfile.bloom.size</name>
+  <value>1048576</value>
+  <description>The size of BloomFilter-s used in BloomMapFile. Each time this many
+  keys is appended the next BloomFilter will be created (inside a DynamicBloomFilter).
+  Larger values minimize the number of filters, which slightly increases the performance,
+  but may waste too much space if the total number of keys is usually much smaller
+  than this number.
+  </description>
+</property>
+
+<property>
+  <name>io.mapfile.bloom.error.rate</name>
+  <value>0.005</value>
+  <description>The rate of false positives in BloomFilter-s used in BloomMapFile.
+  As this value decreases, the size of BloomFilter-s increases exponentially. This
+  value is the probability of encountering false positives (default is 0.5%).
+  </description>
+</property>
+
+<property>
+  <name>hadoop.util.hash.type</name>
+  <value>murmur</value>
+  <description>The default implementation of Hash. Currently this can take one of the
+  two values: 'murmur' to select MurmurHash and 'jenkins' to select JenkinsHash.
+  </description>
+</property>
+
+
+<!-- ipc properties -->
+
+<property>
+  <name>ipc.client.idlethreshold</name>
+  <value>4000</value>
+  <description>Defines the threshold number of connections after which
+               connections will be inspected for idleness.
+  </description>
+</property>
+
+<property>
+  <name>ipc.client.kill.max</name>
+  <value>10</value>
+  <description>Defines the maximum number of clients to disconnect in one go.
+  </description>
+</property>
+
+<property>
+  <name>ipc.client.connection.maxidletime</name>
+  <value>10000</value>
+  <description>The maximum time in msec after which a client will bring down the
+               connection to the server.
+  </description>
+</property>
+
+<property>
+  <name>ipc.client.connect.max.retries</name>
+  <value>10</value>
+  <description>Indicates the number of retries a client will make to establish
+               a server connection.
+  </description>
+</property>
+
+<property>
+  <name>ipc.server.listen.queue.size</name>
+  <value>128</value>
+  <description>Indicates the length of the listen queue for servers accepting
+               client connections.
+  </description>
+</property>
+
+<property>
+  <name>ipc.server.tcpnodelay</name>
+  <value>false</value>
+  <description>Turn on/off Nagle's algorithm for the TCP socket connection on 
+  the server. Setting to true disables the algorithm and may decrease latency
+  with a cost of more/smaller packets. 
+  </description>
+</property>
+
+<property>
+  <name>ipc.client.tcpnodelay</name>
+  <value>false</value>
+  <description>Turn on/off Nagle's algorithm for the TCP socket connection on 
+  the client. Setting to true disables the algorithm and may decrease latency
+  with a cost of more/smaller packets. 
+  </description>
+</property>
+
+
+<!-- Web Interface Configuration -->
+
+<property>
+  <name>webinterface.private.actions</name>
+  <value>false</value>
+  <description> If set to true, the web interfaces of JT and NN may contain 
+                actions, such as kill job, delete file, etc., that should 
+                not be exposed to public. Enable this option if the interfaces 
+                are only reachable by those who have the right authorization.
+  </description>
+</property>
+
+<!-- Proxy Configuration -->
+
+<property>
+  <name>hadoop.rpc.socket.factory.class.default</name>
+  <value>org.apache.hadoop.net.StandardSocketFactory</value>
+  <description> Default SocketFactory to use. This parameter is expected to be
+    formatted as "package.FactoryClassName".
+  </description>
+</property>
+
+<property>
+  <name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
+  <value></value>
+  <description> SocketFactory to use to connect to a DFS. If null or empty, use
+    hadoop.rpc.socket.class.default. This socket factory is also used by
+    DFSClient to create sockets to DataNodes.
+  </description>
+</property>
+
+
+
+<property>
+  <name>hadoop.socks.server</name>
+  <value></value>
+  <description> Address (host:port) of the SOCKS server to be used by the
+    SocksSocketFactory.
+  </description>
+</property>
+
+<!-- Rack Configuration -->
+
+<property>
+  <name>topology.node.switch.mapping.impl</name>
+  <value>org.apache.hadoop.net.ScriptBasedMapping</value>
+  <description> The default implementation of the DNSToSwitchMapping. It
+    invokes a script specified in topology.script.file.name to resolve
+    node names. If the value for topology.script.file.name is not set, the
+    default value of DEFAULT_RACK is returned for all node names.
+  </description>
+</property>
+
+<property>
+  <name>topology.script.file.name</name>
+  <value></value>
+  <description> The script name that should be invoked to resolve DNS names to
+    NetworkTopology names. Example: the script would take host.foo.bar as an
+    argument, and return /rack1 as the output.
+  </description>
+</property>
+
+<property>
+  <name>topology.script.number.args</name>
+  <value>100</value>
+  <description> The max number of args that the script configured with 
+    topology.script.file.name should be run with. Each arg is an
+    IP address.
+  </description>
+</property>
+
+
+
+</configuration>

src/core/org/apache/hadoop/conf/Configuration.java

@@ -41,6 +41,7 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.StringTokenizer;
+import java.util.WeakHashMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -81,9 +82,9 @@ import org.xml.sax.SAXException;
  *
  * <p>Unless explicitly turned off, Hadoop by default specifies two 
  * resources, loaded in-order from the classpath: <ol>
- * <li><tt><a href="{@docRoot}/../hadoop-default.html">hadoop-default.xml</a>
+ * <li><tt><a href="{@docRoot}/../core-default.html">core-default.xml</a>
  * </tt>: Read-only defaults for hadoop.</li>
- * <li><tt>hadoop-site.xml</tt>: Site-specific configuration for a given hadoop
+ * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
  * installation.</li>
  * </ol>
  * Applications may add additional resources, which are loaded
@@ -103,7 +104,7 @@ import org.xml.sax.SAXException;
  *  &lt;/property&gt;</pre></tt>
  *
  * Administrators typically define parameters as final in 
- * <tt>hadoop-site.xml</tt> for values that user applications may not alter.
+ * <tt>core-site.xml</tt> for values that user applications may not alter.
  *
  * <h4 id="VariableExpansion">Variable Expansion</h4>
  *
@@ -149,6 +150,38 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
    */
   private Set<String> finalParameters = new HashSet<String>();
   
+  private boolean loadDefaults = true;
+  
+  /**
+   * Configurtion objects
+   */
+  private static final WeakHashMap<Configuration,Object> REGISTRY = 
+    new WeakHashMap<Configuration,Object>();
+  
+  /**
+   * List of default Resources. Resources are loaded in the order of the list 
+   * entries
+   */
+  private static final ArrayList<String> defaultResources = 
+    new ArrayList<String>();
+  
+  static{
+    //print deprecation warning if hadoop-site.xml is found in classpath
+    ClassLoader cL = Thread.currentThread().getContextClassLoader();
+    if (cL == null) {
+      cL = Configuration.class.getClassLoader();
+    }
+    if(cL.getResource("hadoop-site.xml")!=null) {
+      LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
+          "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
+          + "mapred-site.xml and hdfs-site.xml to override properties of " +
+          "core-default.xml, mapred-default.xml and hdfs-default.xml " +
+          "respectively");
+    }
+    addDefaultResource("core-default.xml");
+    addDefaultResource("core-site.xml");
+  }
+  
   private Properties properties;
   private Properties overlay;
   private ClassLoader classLoader;
@@ -172,12 +205,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
    * @param loadDefaults specifies whether to load from the default files
    */
   public Configuration(boolean loadDefaults) {
+    this.loadDefaults = loadDefaults;
     if (LOG.isDebugEnabled()) {
       LOG.debug(StringUtils.stringifyException(new IOException("config()")));
     }
-    if (loadDefaults) {
-      resources.add("hadoop-default.xml");
-      resources.add("hadoop-site.xml");
+    synchronized(Configuration.class) {
+      REGISTRY.put(this, null);
     }
   }
   
@@ -205,6 +238,25 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
    }
    
     this.finalParameters = new HashSet<String>(other.finalParameters);
+    synchronized(Configuration.class) {
+      REGISTRY.put(this, null);
+    }
+  }
+  
+  /**
+   * Add a default resource. Resources are loaded in the order of the resources 
+   * added.
+   * @param name file name. File should be present in the classpath.
+   */
+  public static synchronized void addDefaultResource(String name) {
+    if(!defaultResources.contains(name)) {
+      defaultResources.add(name);
+      for(Configuration conf : REGISTRY.keySet()) {
+        if(conf.loadDefaults) {
+          conf.reloadConfiguration();
+        }
+      }
+    }
   }
 
   /**
@@ -972,6 +1024,17 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   private void loadResources(Properties properties,
                              ArrayList resources,
                              boolean quiet) {
+    if(loadDefaults) {
+      for (String resource : defaultResources) {
+        loadResource(properties, resource, quiet);
+      }
+    
+      //support the hadoop-site.xml as a deprecated case
+      if(getResource("hadoop-site.xml")!=null) {
+        loadResource(properties, "hadoop-site.xml", quiet);
+      }
+    }
+    
     for (Object resource : resources) {
       loadResource(properties, resource, quiet);
     }
@@ -1158,6 +1221,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   public String toString() {
     StringBuffer sb = new StringBuffer();
     sb.append("Configuration: ");
+    if(loadDefaults) {
+      toString(defaultResources, sb);
+      if(resources.size()>0) {
+        sb.append(", ");
+      }
+    }
     toString(resources, sb);
     return sb.toString();
   }

src/core/org/apache/hadoop/fs/FsShell.java

@@ -1256,9 +1256,8 @@ public class FsShell extends Configured implements Tool {
     String fs = "-fs [local | <file system URI>]: \tSpecify the file system to use.\n" + 
       "\t\tIf not specified, the current configuration is used, \n" +
       "\t\ttaken from the following, in increasing precedence: \n" + 
-      "\t\t\thadoop-default.xml inside the hadoop jar file \n" +
-      "\t\t\thadoop-default.xml in $HADOOP_CONF_DIR \n" +
-      "\t\t\thadoop-site.xml in $HADOOP_CONF_DIR \n" +
+      "\t\t\tcore-default.xml inside the hadoop jar file \n" +
+      "\t\t\tcore-site.xml in $HADOOP_CONF_DIR \n" +
       "\t\t'local' means use the local file system as your DFS. \n" +
       "\t\t<file system URI> specifies a particular file system to \n" +
       "\t\tcontact. This argument is optional but if used must appear\n" +

src/hdfs/hdfs-default.xml

@@ -0,0 +1,356 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Do not modify this file directly.  Instead, copy entries that you -->
+<!-- wish to modify from this file into hdfs-site.xml and change them -->
+<!-- there.  If hdfs-site.xml does not already exist, create it.      -->
+
+<configuration>
+
+<property>
+  <name>dfs.namenode.logging.level</name>
+  <value>info</value>
+  <description>The logging level for dfs namenode. Other values are "dir"(trac
+e namespace mutations), "block"(trace block under/over replications and block
+creations/deletions), or "all".</description>
+</property>
+
+<property>
+  <name>dfs.secondary.http.address</name>
+  <value>0.0.0.0:50090</value>
+  <description>
+    The secondary namenode http server address and port.
+    If the port is 0 then the server will start on a free port.
+  </description>
+</property>
+
+<property>
+  <name>dfs.datanode.address</name>
+  <value>0.0.0.0:50010</value>
+  <description>
+    The address where the datanode server will listen to.
+    If the port is 0 then the server will start on a free port.
+  </description>
+</property>
+
+<property>
+  <name>dfs.datanode.http.address</name>
+  <value>0.0.0.0:50075</value>
+  <description>
+    The datanode http server address and port.
+    If the port is 0 then the server will start on a free port.
+  </description>
+</property>
+
+<property>
+  <name>dfs.datanode.ipc.address</name>
+  <value>0.0.0.0:50020</value>
+  <description>
+    The datanode ipc server address and port.
+    If the port is 0 then the server will start on a free port.
+  </description>
+</property>
+
+<property>
+  <name>dfs.datanode.handler.count</name>
+  <value>3</value>
+  <description>The number of server threads for the datanode.</description>
+</property>
+
+<property>
+  <name>dfs.http.address</name>
+  <value>0.0.0.0:50070</value>
+  <description>
+    The address and the base port where the dfs namenode web ui will listen on.
+    If the port is 0 then the server will start on a free port.
+  </description>
+</property>
+
+<property>
+  <name>dfs.https.enable</name>
+  <value>false</value>
+  <description>Decide if HTTPS(SSL) is supported on HDFS
+  </description>
+</property>
+
+<property>
+  <name>dfs.https.need.client.auth</name>
+  <value>false</value>
+  <description>Whether SSL client certificate authentication is required
+  </description>
+</property>
+
+<property>
+  <name>dfs.https.server.keystore.resource</name>
+  <value>ssl-server.xml</value>
+  <description>Resource file from which ssl server keystore
+  information will be extracted
+  </description>
+</property>
+
+<property>
+  <name>dfs.https.client.keystore.resource</name>
+  <value>ssl-client.xml</value>
+  <description>Resource file from which ssl client keystore
+  information will be extracted
+  </description>
+</property>
+
+<property>
+  <name>dfs.datanode.https.address</name>
+  <value>0.0.0.0:50475</value>
+</property>
+
+<property>
+  <name>dfs.https.address</name>
+  <value>0.0.0.0:50470</value>
+</property>
+
+ <property>
+  <name>dfs.datanode.dns.interface</name>
+  <value>default</value>
+  <description>The name of the Network Interface from which a data node should 
+  report its IP address.
+  </description>
+ </property>
+ 
+<property>
+  <name>dfs.datanode.dns.nameserver</name>
+  <value>default</value>
+  <description>The host name or IP address of the name server (DNS)
+  which a DataNode should use to determine the host name used by the
+  NameNode for communication and display purposes.
+  </description>
+ </property>
+ 
+ 
+ 
+<property>
+  <name>dfs.replication.considerLoad</name>
+  <value>true</value>
+  <description>Decide if chooseTarget considers the target's load or not
+  </description>
+</property>
+<property>
+  <name>dfs.default.chunk.view.size</name>
+  <value>32768</value>
+  <description>The number of bytes to view for a file on the browser.
+  </description>
+</property>
+
+<property>
+  <name>dfs.datanode.du.reserved</name>
+  <value>0</value>
+  <description>Reserved space in bytes per volume. Always leave this much space free for non dfs use.
+  </description>
+</property>
+
+<property>
+  <name>dfs.name.dir</name>
+  <value>${hadoop.tmp.dir}/dfs/name</value>
+  <description>Determines where on the local filesystem the DFS name node
+      should store the name table(fsimage).  If this is a comma-delimited list
+      of directories then the name table is replicated in all of the
+      directories, for redundancy. </description>
+</property>
+
+<property>
+  <name>dfs.name.edits.dir</name>
+  <value>${dfs.name.dir}</value>
+  <description>Determines where on the local filesystem the DFS name node
+      should store the transaction (edits) file. If this is a comma-delimited list
+      of directories then the transaction file is replicated in all of the 
+      directories, for redundancy. Default value is same as dfs.name.dir
+  </description>
+</property>
+<property>
+  <name>dfs.web.ugi</name>
+  <value>webuser,webgroup</value>
+  <description>The user account used by the web interface.
+    Syntax: USERNAME,GROUP1,GROUP2, ...
+  </description>
+</property>
+
+<property>
+  <name>dfs.permissions</name>
+  <value>true</value>
+  <description>
+    If "true", enable permission checking in HDFS.
+    If "false", permission checking is turned off,
+    but all other behavior is unchanged.
+    Switching from one parameter value to the other does not change the mode,
+    owner or group of files or directories.
+  </description>
+</property>
+
+<property>
+  <name>dfs.permissions.supergroup</name>
+  <value>supergroup</value>
+  <description>The name of the group of super-users.</description>
+</property>
+
+<property>
+  <name>dfs.data.dir</name>
+  <value>${hadoop.tmp.dir}/dfs/data</value>
+  <description>Determines where on the local filesystem an DFS data node
+  should store its blocks.  If this is a comma-delimited
+  list of directories, then data will be stored in all named
+  directories, typically on different devices.
+  Directories that do not exist are ignored.
+  </description>
+</property>
+
+<property>
+  <name>dfs.replication</name>
+  <value>3</value>
+  <description>Default block replication. 
+  The actual number of replications can be specified when the file is created.
+  The default is used if replication is not specified in create time.
+  </description>
+</property>
+
+<property>
+  <name>dfs.replication.max</name>
+  <value>512</value>
+  <description>Maximal block replication. 
+  </description>
+</property>
+
+<property>
+  <name>dfs.replication.min</name>
+  <value>1</value>
+  <description>Minimal block replication. 
+  </description>
+</property>
+
+<property>
+  <name>dfs.block.size</name>
+  <value>67108864</value>
+  <description>The default block size for new files.</description>
+</property>
+
+<property>
+  <name>dfs.df.interval</name>
+  <value>60000</value>
+  <description>Disk usage statistics refresh interval in msec.</description>
+</property>
+
+<property>
+  <name>dfs.client.block.write.retries</name>
+  <value>3</value>
+  <description>The number of retries for writing blocks to the data nodes, 
+  before we signal failure to the application.
+  </description>
+</property>
+
+<property>
+  <name>dfs.blockreport.intervalMsec</name>
+  <value>3600000</value>
+  <description>Determines block reporting interval in milliseconds.</description>
+</property>
+
+<property>
+  <name>dfs.blockreport.initialDelay</name>  <value>0</value>
+  <description>Delay for first block report in seconds.</description>
+</property>
+
+<property>
+  <name>dfs.heartbeat.interval</name>
+  <value>3</value>
+  <description>Determines datanode heartbeat interval in seconds.</description>
+</property>
+
+<property>
+  <name>dfs.namenode.handler.count</name>
+  <value>10</value>
+  <description>The number of server threads for the namenode.</description>
+</property>
+
+<property>
+  <name>dfs.safemode.threshold.pct</name>
+  <value>0.999f</value>
+  <description>
+    Specifies the percentage of blocks that should satisfy 
+    the minimal replication requirement defined by dfs.replication.min.
+    Values less than or equal to 0 mean not to start in safe mode.
+    Values greater than 1 will make safe mode permanent.
+  </description>
+</property>
+
+<property>
+  <name>dfs.safemode.extension</name>
+  <value>30000</value>
+  <description>
+    Determines extension of safe mode in milliseconds 
+    after the threshold level is reached.
+  </description>
+</property>
+
+<property>
+  <name>dfs.balance.bandwidthPerSec</name>
+  <value>1048576</value>
+  <description>
+        Specifies the maximum amount of bandwidth that each datanode
+        can utilize for the balancing purpose in term of
+        the number of bytes per second.
+  </description>
+</property>
+
+<property>
+  <name>dfs.hosts</name>
+  <value></value>
+  <description>Names a file that contains a list of hosts that are
+  permitted to connect to the namenode. The full pathname of the file
+  must be specified.  If the value is empty, all hosts are
+  permitted.</description>
+</property>
+
+<property>
+  <name>dfs.hosts.exclude</name>
+  <value></value>
+  <description>Names a file that contains a list of hosts that are
+  not permitted to connect to the namenode.  The full pathname of the
+  file must be specified.  If the value is empty, no hosts are
+  excluded.</description>
+</property> 
+
+<property>
+  <name>dfs.max.objects</name>
+  <value>0</value>
+  <description>The maximum number of files, directories and blocks
+  dfs supports. A value of zero indicates no limit to the number
+  of objects that dfs supports.
+  </description>
+</property>
+
+<property>
+  <name>dfs.namenode.decommission.interval</name>
+  <value>30</value>
+  <description>Namenode periodicity in seconds to check if decommission is 
+  complete.</description>
+</property>
+
+<property>
+  <name>dfs.namenode.decommission.nodes.per.interval</name>
+  <value>5</value>
+  <description>The number of nodes namenode checks if decommission is complete
+  in each dfs.namenode.decommission.interval.</description>
+</property>
+
+<property>
+  <name>dfs.replication.interval</name>
+  <value>3</value>
+  <description>The periodicity in seconds with which the namenode computes 
+  repliaction work for datanodes. </description>
+</property>
+
+<property>
+  <name>dfs.access.time.precision</name>
+  <value>3600000</value>
+  <description>The access time for HDFS file is precise upto this value. 
+               The default value is 1 hour. Setting a value of 0 disables
+               access times for HDFS.
+  </description>
+</property>
+
+</configuration>

src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -49,6 +49,11 @@ public class DistributedFileSystem extends FileSystem {
 
   DFSClient dfs;
   private boolean verifyChecksum = true;
+  
+  static{
+    Configuration.addDefaultResource("hdfs-default.xml");
+    Configuration.addDefaultResource("hdfs-site.xml");
+  }
 
   public DistributedFileSystem() {
   }
@@ -449,4 +454,6 @@ public class DistributedFileSystem extends FileSystem {
       ) throws IOException {
     dfs.setTimes(getPathName(p), mtime, atime);
   }
+  
+  
 }

src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java

@@ -126,6 +126,11 @@ import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
 public class DataNode extends Configured 
     implements InterDatanodeProtocol, ClientDatanodeProtocol, FSConstants, Runnable {
   public static final Log LOG = LogFactory.getLog(DataNode.class);
+  
+  static{
+    Configuration.addDefaultResource("hdfs-default.xml");
+    Configuration.addDefaultResource("hdfs-site.xml");
+  }
 
   public static final String DN_CLIENTTRACE_FORMAT =
         "src: %s" +      // src IP

src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java

@@ -97,6 +97,11 @@ import java.util.Iterator;
 public class NameNode implements ClientProtocol, DatanodeProtocol,
                                  NamenodeProtocol, FSConstants,
                                  RefreshAuthorizationPolicyProtocol {
+  static{
+    Configuration.addDefaultResource("hdfs-default.xml");
+    Configuration.addDefaultResource("hdfs-site.xml");
+  }
+  
   public long getProtocolVersion(String protocol, 
                                  long clientVersion) throws IOException { 
     if (protocol.equals(ClientProtocol.class.getName())) {

conf/hadoop-default.xml → src/mapred/mapred-default.xml

@@ -2,55 +2,11 @@
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 
 <!-- Do not modify this file directly.  Instead, copy entries that you -->
-<!-- wish to modify from this file into hadoop-site.xml and change them -->
-<!-- there.  If hadoop-site.xml does not already exist, create it.      -->
+<!-- wish to modify from this file into mapred-site.xml and change them -->
+<!-- there.  If mapred-site.xml does not already exist, create it.      -->
 
 <configuration>
 
-<!--- global properties -->
-
-<property>
-  <name>hadoop.tmp.dir</name>
-  <value>/tmp/hadoop-${user.name}</value>
-  <description>A base for other temporary directories.</description>
-</property>
-
-<property>
-  <name>hadoop.native.lib</name>
-  <value>true</value>
-  <description>Should native hadoop libraries, if present, be used.</description>
-</property>
-
-<property>
-  <name>hadoop.http.filter.initializers</name>
-  <value></value>
-  <description>A comma separated list of class names. Each class in the list 
-  must extend org.apache.hadoop.http.FilterInitializer. The corresponding 
-  Filter will be initialized. Then, the Filter will be applied to all user 
-  facing jsp and servlet web pages.  The ordering of the list defines the 
-  ordering of the filters.</description>
-</property>
-
-<property>
-  <name>hadoop.security.authorization</name>
-  <value>false</value>
-  <description>Is service-level authorization enabled?</description>
-</property>
-
-<!--- logging properties -->
-
-<property>
-  <name>hadoop.logfile.size</name>
-  <value>10000000</value>
-  <description>The max size of each log file</description>
-</property>
-
-<property>
-  <name>hadoop.logfile.count</name>
-  <value>10</value>
-  <description>The max number of log files</description>
-</property>
-
 <property>
   <name>hadoop.job.history.location</name>
   <value></value>
@@ -70,14 +26,6 @@
   </description>
 </property>
 
-<property>
-  <name>dfs.namenode.logging.level</name>
-  <value>info</value>
-  <description>The logging level for dfs namenode. Other values are "dir"(trac
-e namespace mutations), "block"(trace block under/over replications and block
-creations/deletions), or "all".</description>
-</property>
-
 <!-- i/o properties -->
 
 <property>
@@ -113,30 +61,6 @@ creations/deletions), or "all".</description>
   the spill. A value less than 0.5 is not recommended.</description>
 </property>
 
-<property>
-  <name>io.file.buffer.size</name>
-  <value>4096</value>
-  <description>The size of buffer for use in sequence files.
-  The size of this buffer should probably be a multiple of hardware
-  page size (4096 on Intel x86), and it determines how much data is
-  buffered during read and write operations.</description>
-</property>
-  
-<property>
-  <name>io.bytes.per.checksum</name>
-  <value>512</value>
-  <description>The number of bytes per checksum.  Must not be larger than
-  io.file.buffer.size.</description>
-</property>
-
-<property>
-  <name>io.skip.checksum.errors</name>
-  <value>false</value>
-  <description>If true, when a checksum error is encountered while
-  reading a sequence file, entries are skipped, instead of throwing an
-  exception.</description>
-</property>
-  
 <property>
   <name>io.map.index.skip</name>
   <value>0</value>
@@ -145,502 +69,6 @@ creations/deletions), or "all".</description>
   facilitate opening large map files using less memory.</description>
 </property>
 
-<property>
-  <name>io.compression.codecs</name>
-  <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec</value>
-  <description>A list of the compression codec classes that can be used 
-               for compression/decompression.</description>
-</property>
-
-<property>
-  <name>io.serializations</name>
-  <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
-  <description>A list of serialization classes that can be used for
-  obtaining serializers and deserializers.</description>
-</property>
-
-<!-- file system properties -->
-
-<property>
-  <name>fs.default.name</name>
-  <value>file:///</value>
-  <description>The name of the default file system.  A URI whose
-  scheme and authority determine the FileSystem implementation.  The
-  uri's scheme determines the config property (fs.SCHEME.impl) naming
-  the FileSystem implementation class.  The uri's authority is used to
-  determine the host, port, etc. for a filesystem.</description>
-</property>
-
-<property>
-  <name>fs.trash.interval</name>
-  <value>0</value>
-  <description>Number of minutes between trash checkpoints.
-  If zero, the trash feature is disabled.
-  </description>
-</property>
-
-<property>
-  <name>fs.file.impl</name>
-  <value>org.apache.hadoop.fs.LocalFileSystem</value>
-  <description>The FileSystem for file: uris.</description>
-</property>
-
-<property>
-  <name>fs.hdfs.impl</name>
-  <value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
-  <description>The FileSystem for hdfs: uris.</description>
-</property>
-
-<property>
-  <name>fs.s3.impl</name>
-  <value>org.apache.hadoop.fs.s3.S3FileSystem</value>
-  <description>The FileSystem for s3: uris.</description>
-</property>
-
-<property>
-  <name>fs.s3n.impl</name>
-  <value>org.apache.hadoop.fs.s3native.NativeS3FileSystem</value>
-  <description>The FileSystem for s3n: (Native S3) uris.</description>
-</property>
-
-<property>
-  <name>fs.kfs.impl</name>
-  <value>org.apache.hadoop.fs.kfs.KosmosFileSystem</value>
-  <description>The FileSystem for kfs: uris.</description>
-</property>
-
-<property>
-  <name>fs.hftp.impl</name>
-  <value>org.apache.hadoop.hdfs.HftpFileSystem</value>
-</property>
-
-<property>
-  <name>fs.hsftp.impl</name>
-  <value>org.apache.hadoop.hdfs.HsftpFileSystem</value>
-</property>
-
-<property>
-  <name>fs.ftp.impl</name>
-  <value>org.apache.hadoop.fs.ftp.FTPFileSystem</value>
-  <description>The FileSystem for ftp: uris.</description>
-</property>
-
-<property>
-  <name>fs.ramfs.impl</name>
-  <value>org.apache.hadoop.fs.InMemoryFileSystem</value>
-  <description>The FileSystem for ramfs: uris.</description>
-</property>
-
-<property>
-  <name>fs.har.impl</name>
-  <value>org.apache.hadoop.fs.HarFileSystem</value>
-  <description>The filesystem for Hadoop archives. </description>
-</property>
-
-<property>
-  <name>fs.checkpoint.dir</name>
-  <value>${hadoop.tmp.dir}/dfs/namesecondary</value>
-  <description>Determines where on the local filesystem the DFS secondary
-      name node should store the temporary images to merge.
-      If this is a comma-delimited list of directories then the image is
-      replicated in all of the directories for redundancy.
-  </description>
-</property>
-
-<property>
-  <name>fs.checkpoint.edits.dir</name>
-  <value>${fs.checkpoint.dir}</value>
-  <description>Determines where on the local filesystem the DFS secondary
-      name node should store the temporary edits to merge.
-      If this is a comma-delimited list of directoires then teh edits is
-      replicated in all of the directoires for redundancy.
-      Default value is same as fs.checkpoint.dir
-  </description>
-</property>
-
-<property>
-  <name>fs.checkpoint.period</name>
-  <value>3600</value>
-  <description>The number of seconds between two periodic checkpoints.
-  </description>
-</property>
-
-<property>
-  <name>fs.checkpoint.size</name>
-  <value>67108864</value>
-  <description>The size of the current edit log (in bytes) that triggers
-       a periodic checkpoint even if the fs.checkpoint.period hasn't expired.
-  </description>
-</property>
-
-<property>
-  <name>dfs.secondary.http.address</name>
-  <value>0.0.0.0:50090</value>
-  <description>
-    The secondary namenode http server address and port.
-    If the port is 0 then the server will start on a free port.
-  </description>
-</property>
-
-<property>
-  <name>dfs.datanode.address</name>
-  <value>0.0.0.0:50010</value>
-  <description>
-    The address where the datanode server will listen to.
-    If the port is 0 then the server will start on a free port.
-  </description>
-</property>
-
-<property>
-  <name>dfs.datanode.http.address</name>
-  <value>0.0.0.0:50075</value>
-  <description>
-    The datanode http server address and port.
-    If the port is 0 then the server will start on a free port.
-  </description>
-</property>
-
-<property>
-  <name>dfs.datanode.ipc.address</name>
-  <value>0.0.0.0:50020</value>
-  <description>
-    The datanode ipc server address and port.
-    If the port is 0 then the server will start on a free port.
-  </description>
-</property>
-
-<property>
-  <name>dfs.datanode.handler.count</name>
-  <value>3</value>
-  <description>The number of server threads for the datanode.</description>
-</property>
-
-<property>
-  <name>dfs.http.address</name>
-  <value>0.0.0.0:50070</value>
-  <description>
-    The address and the base port where the dfs namenode web ui will listen on.
-    If the port is 0 then the server will start on a free port.
-  </description>
-</property>
-
-<property>
-  <name>dfs.https.enable</name>
-  <value>false</value>
-  <description>Decide if HTTPS(SSL) is supported on HDFS
-  </description>
-</property>
-
-<property>
-  <name>dfs.https.need.client.auth</name>
-  <value>false</value>
-  <description>Whether SSL client certificate authentication is required
-  </description>
-</property>
-
-<property>
-  <name>dfs.https.server.keystore.resource</name>
-  <value>ssl-server.xml</value>
-  <description>Resource file from which ssl server keystore
-  information will be extracted
-  </description>
-</property>
-
-<property>
-  <name>dfs.https.client.keystore.resource</name>
-  <value>ssl-client.xml</value>
-  <description>Resource file from which ssl client keystore
-  information will be extracted
-  </description>
-</property>
-
-<property>
-  <name>dfs.datanode.https.address</name>
-  <value>0.0.0.0:50475</value>
-</property>
-
-<property>
-  <name>dfs.https.address</name>
-  <value>0.0.0.0:50470</value>
-</property>
-
- <property>
-  <name>dfs.datanode.dns.interface</name>
-  <value>default</value>
-  <description>The name of the Network Interface from which a data node should 
-  report its IP address.
-  </description>
- </property>
- 
-<property>
-  <name>dfs.datanode.dns.nameserver</name>
-  <value>default</value>
-  <description>The host name or IP address of the name server (DNS)
-  which a DataNode should use to determine the host name used by the
-  NameNode for communication and display purposes.
-  </description>
- </property>
- 
-<property>
-  <name>dfs.replication.considerLoad</name>
-  <value>true</value>
-  <description>Decide if chooseTarget considers the target's load or not
-  </description>
-</property>
-<property>
-  <name>dfs.default.chunk.view.size</name>
-  <value>32768</value>
-  <description>The number of bytes to view for a file on the browser.
-  </description>
-</property>
-
-<property>
-  <name>dfs.datanode.du.reserved</name>
-  <value>0</value>
-  <description>Reserved space in bytes per volume. Always leave this much space free for non dfs use.
-  </description>
-</property>
-
-<property>
-  <name>dfs.name.dir</name>
-  <value>${hadoop.tmp.dir}/dfs/name</value>
-  <description>Determines where on the local filesystem the DFS name node
-      should store the name table(fsimage).  If this is a comma-delimited list
-      of directories then the name table is replicated in all of the
-      directories, for redundancy. </description>
-</property>
-
-<property>
-  <name>dfs.name.edits.dir</name>
-  <value>${dfs.name.dir}</value>
-  <description>Determines where on the local filesystem the DFS name node
-      should store the transaction (edits) file. If this is a comma-delimited list
-      of directories then the transaction file is replicated in all of the 
-      directories, for redundancy. Default value is same as dfs.name.dir
-  </description>
-</property>
-<property>
-  <name>dfs.web.ugi</name>
-  <value>webuser,webgroup</value>
-  <description>The user account used by the web interface.
-    Syntax: USERNAME,GROUP1,GROUP2, ...
-  </description>
-</property>
-
-<property>
-  <name>dfs.permissions</name>
-  <value>true</value>
-  <description>
-    If "true", enable permission checking in HDFS.
-    If "false", permission checking is turned off,
-    but all other behavior is unchanged.
-    Switching from one parameter value to the other does not change the mode,
-    owner or group of files or directories.
-  </description>
-</property>
-
-<property>
-  <name>dfs.permissions.supergroup</name>
-  <value>supergroup</value>
-  <description>The name of the group of super-users.</description>
-</property>
-
-<property>
-  <name>dfs.data.dir</name>
-  <value>${hadoop.tmp.dir}/dfs/data</value>
-  <description>Determines where on the local filesystem an DFS data node
-  should store its blocks.  If this is a comma-delimited
-  list of directories, then data will be stored in all named
-  directories, typically on different devices.
-  Directories that do not exist are ignored.
-  </description>
-</property>
-
-<property>
-  <name>dfs.replication</name>
-  <value>3</value>
-  <description>Default block replication. 
-  The actual number of replications can be specified when the file is created.
-  The default is used if replication is not specified in create time.
-  </description>
-</property>
-
-<property>
-  <name>dfs.replication.max</name>
-  <value>512</value>
-  <description>Maximal block replication. 
-  </description>
-</property>
-
-<property>
-  <name>dfs.replication.min</name>
-  <value>1</value>
-  <description>Minimal block replication. 
-  </description>
-</property>
-
-<property>
-  <name>dfs.block.size</name>
-  <value>67108864</value>
-  <description>The default block size for new files.</description>
-</property>
-
-<property>
-  <name>dfs.df.interval</name>
-  <value>60000</value>
-  <description>Disk usage statistics refresh interval in msec.</description>
-</property>
-
-<property>
-  <name>dfs.client.block.write.retries</name>
-  <value>3</value>
-  <description>The number of retries for writing blocks to the data nodes, 
-  before we signal failure to the application.
-  </description>
-</property>
-
-<property>
-  <name>dfs.blockreport.intervalMsec</name>
-  <value>3600000</value>
-  <description>Determines block reporting interval in milliseconds.</description>
-</property>
-
-<property>
-  <name>dfs.blockreport.initialDelay</name>  <value>0</value>
-  <description>Delay for first block report in seconds.</description>
-</property>
-
-<property>
-  <name>dfs.heartbeat.interval</name>
-  <value>3</value>
-  <description>Determines datanode heartbeat interval in seconds.</description>
-</property>
-
-<property>
-  <name>dfs.namenode.handler.count</name>
-  <value>10</value>
-  <description>The number of server threads for the namenode.</description>
-</property>
-
-<property>
-  <name>dfs.safemode.threshold.pct</name>
-  <value>0.999f</value>
-  <description>
-  	Specifies the percentage of blocks that should satisfy 
-  	the minimal replication requirement defined by dfs.replication.min.
-  	Values less than or equal to 0 mean not to start in safe mode.
-  	Values greater than 1 will make safe mode permanent.
- 	</description>
-</property>
-
-<property>
-  <name>dfs.safemode.extension</name>
-  <value>30000</value>
-  <description>
-  	Determines extension of safe mode in milliseconds 
-  	after the threshold level is reached.
- 	</description>
-</property>
-
-<property>
-  <name>dfs.balance.bandwidthPerSec</name>
-  <value>1048576</value>
-  <description>
-        Specifies the maximum amount of bandwidth that each datanode
-        can utilize for the balancing purpose in term of
-        the number of bytes per second.
-  </description>
-</property>
-
-<property>
-  <name>dfs.hosts</name>
-  <value></value>
-  <description>Names a file that contains a list of hosts that are
-  permitted to connect to the namenode. The full pathname of the file
-  must be specified.  If the value is empty, all hosts are
-  permitted.</description>
-</property>
-
-<property>
-  <name>dfs.hosts.exclude</name>
-  <value></value>
-  <description>Names a file that contains a list of hosts that are
-  not permitted to connect to the namenode.  The full pathname of the
-  file must be specified.  If the value is empty, no hosts are
-  excluded.</description>
-</property> 
-
-<property>
-  <name>dfs.max.objects</name>
-  <value>0</value>
-  <description>The maximum number of files, directories and blocks
-  dfs supports. A value of zero indicates no limit to the number
-  of objects that dfs supports.
-  </description>
-</property>
-
-<property>
-  <name>dfs.namenode.decommission.interval</name>
-  <value>30</value>
-  <description>Namenode periodicity in seconds to check if decommission is 
-  complete.</description>
-</property>
-
-<property>
-  <name>dfs.namenode.decommission.nodes.per.interval</name>
-  <value>5</value>
-  <description>The number of nodes namenode checks if decommission is complete
-  in each dfs.namenode.decommission.interval.</description>
-</property>
-
-<property>
-  <name>dfs.replication.interval</name>
-  <value>3</value>
-  <description>The periodicity in seconds with which the namenode computes 
-  repliaction work for datanodes. </description>
-</property>
-
-<property>
-  <name>dfs.access.time.precision</name>
-  <value>3600000</value>
-  <description>The access time for HDFS file is precise upto this value. 
-               The default value is 1 hour. Setting a value of 0 disables
-               access times for HDFS.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3.block.size</name>
-  <value>67108864</value>
-  <description>Block size to use when writing files to S3.</description>
-</property>
-
-<property>
-  <name>fs.s3.buffer.dir</name>
-  <value>${hadoop.tmp.dir}/s3</value>
-  <description>Determines where on the local filesystem the S3 filesystem
-  should store files before sending them to S3
-  (or after retrieving them from S3).
-  </description>
-</property>
-
-<property>
-  <name>fs.s3.maxRetries</name>
-  <value>4</value>
-  <description>The maximum number of retries for reading or writing files to S3, 
-  before we signal failure to the application.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3.sleepTimeSeconds</name>
-  <value>10</value>
-  <description>The number of seconds to sleep between each S3 retry.
-  </description>
-</property>
-
-<!-- map/reduce properties -->
-
 <property>
   <name>mapred.job.tracker</name>
   <value>local</value>
@@ -687,14 +115,6 @@ creations/deletions), or "all".</description>
   </description>
 </property>
 
-<property>
-  <name>local.cache.size</name>
-  <value>10737418240</value>
-  <description>The limit on the size of cache you want to keep, set by default
-  to 10GB. This will act as a soft limit on the cache directory for out of band data.
-  </description>
-</property>
-            
 <property>
   <name>mapred.system.dir</name>
   <value>${hadoop.tmp.dir}/mapred/system</value>
@@ -722,11 +142,11 @@ creations/deletions), or "all".</description>
   <name>mapred.local.dir.minspacekill</name>
   <value>0</value>
   <description>If the space in mapred.local.dir drops under this, 
-  	do not ask more tasks until all the current ones have finished and 
-  	cleaned up. Also, to save the rest of the tasks we have running, 
-  	kill one of them, to clean up some space. Start with the reduce tasks,
-  	then go with the ones that have finished the least.
-  	Value in bytes.
+    do not ask more tasks until all the current ones have finished and 
+    cleaned up. Also, to save the rest of the tasks we have running, 
+    kill one of them, to clean up some space. Start with the reduce tasks,
+    then go with the ones that have finished the least.
+    Value in bytes.
   </description>
 </property>
 
@@ -1192,6 +612,7 @@ creations/deletions), or "all".</description>
                from the reduce directory as they are consumed.</description>
 </property>
 
+
 <!-- 
   <property>
   <name>keep.task.files.pattern</name>
@@ -1239,58 +660,6 @@ creations/deletions), or "all".</description>
   </description>
 </property>
 
-<property>
-  <name>io.seqfile.compress.blocksize</name>
-  <value>1000000</value>
-  <description>The minimum block size for compression in block compressed 
-  				SequenceFiles.
-  </description>
-</property>
-
-<property>
-  <name>io.seqfile.lazydecompress</name>
-  <value>true</value>
-  <description>Should values of block-compressed SequenceFiles be decompressed
-  				only when necessary.
-  </description>
-</property>
-
-<property>
-  <name>io.seqfile.sorter.recordlimit</name>
-  <value>1000000</value>
-  <description>The limit on number of records to be kept in memory in a spill 
-  				in SequenceFiles.Sorter
-  </description>
-</property>
-
-<property>
-  <name>io.mapfile.bloom.size</name>
-  <value>1048576</value>
-  <description>The size of BloomFilter-s used in BloomMapFile. Each time this many
-  keys is appended the next BloomFilter will be created (inside a DynamicBloomFilter).
-  Larger values minimize the number of filters, which slightly increases the performance,
-  but may waste too much space if the total number of keys is usually much smaller
-  than this number.
-  </description>
-</property>
-
-<property>
-  <name>io.mapfile.bloom.error.rate</name>
-  <value>0.005</value>
-  <description>The rate of false positives in BloomFilter-s used in BloomMapFile.
-  As this value decreases, the size of BloomFilter-s increases exponentially. This
-  value is the probability of encountering false positives (default is 0.5%).
-  </description>
-</property>
-
-<property>
-  <name>hadoop.util.hash.type</name>
-  <value>murmur</value>
-  <description>The default implementation of Hash. Currently this can take one of the
-  two values: 'murmur' to select MurmurHash and 'jenkins' to select JenkinsHash.
-  </description>
-</property>
-
 <property>
   <name>map.sort.class</name>
   <value>org.apache.hadoop.util.QuickSort</value>
@@ -1309,7 +678,7 @@ creations/deletions), or "all".</description>
   <name>mapred.userlog.retain.hours</name>
   <value>24</value>
   <description>The maximum time, in hours, for which the user-logs are to be 
-  				retained.
+          retained.
   </description>
 </property>
 
@@ -1327,18 +696,18 @@ creations/deletions), or "all".</description>
   <description>Names a file that contains the list of hosts that
   should be excluded by the jobtracker.  If the value is empty, no
   hosts are excluded.</description>
-</property> 
+</property>
 
 <property>
   <name>mapred.max.tracker.blacklists</name>
   <value>4</value>
-  <description>The number of blacklists for a taskTracker by various jobs 
+  <description>The number of blacklists for a taskTracker by various jobs
                after which the task tracker could be blacklisted across
-               all jobs. The tracker will be given a tasks later 
-               (after a day). The tracker will become a healthy 
-               tracker after a restart. 
+               all jobs. The tracker will be given a tasks later
+               (after a day). The tracker will become a healthy
+               tracker after a restart.
   </description>
-</property>
+</property> 
 
 <property>
   <name>mapred.max.tracker.failures</name>
@@ -1491,66 +860,7 @@ creations/deletions), or "all".</description>
     acceptable.
     </description>
   </property>
-
-<!-- ipc properties -->
-
-<property>
-  <name>ipc.client.idlethreshold</name>
-  <value>4000</value>
-  <description>Defines the threshold number of connections after which
-               connections will be inspected for idleness.
-  </description>
-</property>
-
-<property>
-  <name>ipc.client.kill.max</name>
-  <value>10</value>
-  <description>Defines the maximum number of clients to disconnect in one go.
-  </description>
-</property>
-
-<property>
-  <name>ipc.client.connection.maxidletime</name>
-  <value>10000</value>
-  <description>The maximum time in msec after which a client will bring down the
-               connection to the server.
-  </description>
-</property>
-
-<property>
-  <name>ipc.client.connect.max.retries</name>
-  <value>10</value>
-  <description>Indicates the number of retries a client will make to establish
-               a server connection.
-  </description>
-</property>
-
-<property>
-  <name>ipc.server.listen.queue.size</name>
-  <value>128</value>
-  <description>Indicates the length of the listen queue for servers accepting
-               client connections.
-  </description>
-</property>
-
-<property>
-  <name>ipc.server.tcpnodelay</name>
-  <value>false</value>
-  <description>Turn on/off Nagle's algorithm for the TCP socket connection on 
-  the server. Setting to true disables the algorithm and may decrease latency
-  with a cost of more/smaller packets. 
-  </description>
-</property>
-
-<property>
-  <name>ipc.client.tcpnodelay</name>
-  <value>false</value>
-  <description>Turn on/off Nagle's algorithm for the TCP socket connection on 
-  the client. Setting to true disables the algorithm and may decrease latency
-  with a cost of more/smaller packets. 
-  </description>
-</property>
-
+  
 <!-- Job Notification Configuration -->
 
 <!--
@@ -1579,38 +889,8 @@ creations/deletions), or "all".</description>
    <description>Indicates time in milliseconds between notification URL retry
                 calls</description>
 </property>
-
-<!-- Web Interface Configuration -->
-
-<property>
-  <name>webinterface.private.actions</name>
-  <value>false</value>
-  <description> If set to true, the web interfaces of JT and NN may contain 
-                actions, such as kill job, delete file, etc., that should 
-                not be exposed to public. Enable this option if the interfaces 
-                are only reachable by those who have the right authorization.
-  </description>
-</property>
-
+  
 <!-- Proxy Configuration -->
-
-<property>
-  <name>hadoop.rpc.socket.factory.class.default</name>
-  <value>org.apache.hadoop.net.StandardSocketFactory</value>
-  <description> Default SocketFactory to use. This parameter is expected to be
-    formatted as "package.FactoryClassName".
-  </description>
-</property>
-
-<property>
-  <name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
-  <value></value>
-  <description> SocketFactory to use to connect to a DFS. If null or empty, use
-    hadoop.rpc.socket.class.default. This socket factory is also used by
-    DFSClient to create sockets to DataNodes.
-  </description>
-</property>
-
 <property>
   <name>hadoop.rpc.socket.factory.class.JobSubmissionProtocol</name>
   <value></value>
@@ -1619,44 +899,6 @@ creations/deletions), or "all".</description>
   </description>
 </property>
 
-<property>
-  <name>hadoop.socks.server</name>
-  <value></value>
-  <description> Address (host:port) of the SOCKS server to be used by the
-    SocksSocketFactory.
-  </description>
-</property>
-
-<!-- Rack Configuration -->
-
-<property>
-  <name>topology.node.switch.mapping.impl</name>
-  <value>org.apache.hadoop.net.ScriptBasedMapping</value>
-  <description> The default implementation of the DNSToSwitchMapping. It
-    invokes a script specified in topology.script.file.name to resolve
-    node names. If the value for topology.script.file.name is not set, the
-    default value of DEFAULT_RACK is returned for all node names.
-  </description>
-</property>
-
-<property>
-  <name>topology.script.file.name</name>
-  <value></value>
-  <description> The script name that should be invoked to resolve DNS names to
-    NetworkTopology names. Example: the script would take host.foo.bar as an
-    argument, and return /rack1 as the output.
-  </description>
-</property>
-
-<property>
-  <name>topology.script.number.args</name>
-  <value>100</value>
-  <description> The max number of args that the script configured with 
-    topology.script.file.name should be run with. Each arg is an
-    IP address.
-  </description>
-</property>
-
 <property>
   <name>mapred.task.cache.levels</name>
   <value>2</value>
@@ -1743,4 +985,4 @@ creations/deletions), or "all".</description>
   </description>
 </property>
 
-</configuration>
+</configuration>

src/mapred/org/apache/hadoop/mapred/JobClient.java

@@ -161,6 +161,11 @@ public class JobClient extends Configured implements MRConstants, Tool  {
   private TaskStatusFilter taskOutputFilter = TaskStatusFilter.FAILED; 
   private static final long MAX_JOBPROFILE_AGE = 1000 * 2;
 
+  static{
+    Configuration.addDefaultResource("mapred-default.xml");
+    Configuration.addDefaultResource("mapred-site.xml");
+  }
+
   /**
    * A NetworkedJob is an implementation of RunningJob.  It holds
    * a JobProfile object to provide some info, and interacts with the

src/mapred/org/apache/hadoop/mapred/JobConf.java

@@ -106,6 +106,11 @@ public class JobConf extends Configuration {
   
   private static final Log LOG = LogFactory.getLog(JobConf.class);
 
+  static{
+    Configuration.addDefaultResource("mapred-default.xml");
+    Configuration.addDefaultResource("mapred-site.xml");
+  }
+
   /**
    * A value which if set for memory related configuration options,
    * indicates that the options are turned off.

src/mapred/org/apache/hadoop/mapred/JobTracker.java

@@ -86,6 +86,11 @@ import org.apache.hadoop.util.VersionInfo;
 public class JobTracker implements MRConstants, InterTrackerProtocol,
     JobSubmissionProtocol, TaskTrackerManager, RefreshAuthorizationPolicyProtocol {
 
+  static{
+    Configuration.addDefaultResource("mapred-default.xml");
+    Configuration.addDefaultResource("mapred-site.xml");
+  }
+
   static long TASKTRACKER_EXPIRY_INTERVAL = 10 * 60 * 1000;
   static long RETIRE_JOB_INTERVAL;
   static long RETIRE_JOB_CHECK_INTERVAL;

src/mapred/org/apache/hadoop/mapred/TaskTracker.java

@@ -101,6 +101,11 @@ public class TaskTracker
 
   static enum State {NORMAL, STALE, INTERRUPTED, DENIED}
 
+  static{
+    Configuration.addDefaultResource("mapred-default.xml");
+    Configuration.addDefaultResource("mapred-site.xml");
+  }
+
   public static final Log LOG =
     LogFactory.getLog(TaskTracker.class);
 

src/test/core-site.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Values used when running unit tests.  This is mostly empty, to -->
+<!-- use of the default values, overriding the potentially -->
+<!-- user-editted core-site.xml in the conf/ directory.  -->
+
+<configuration>
+
+
+<property>
+  <name>hadoop.tmp.dir</name>
+  <value>build/test</value>
+  <description>A base for other temporary directories.</description>
+  <final>true</final>
+</property>
+
+<property>
+  <name>test.fs.s3.name</name>
+  <value>s3:///</value>
+  <description>The name of the s3 file system for testing.</description>
+</property>
+
+<property>
+  <name>fs.s3.block.size</name>
+  <value>128</value>
+  <description>Size of a block in bytes.</description>
+</property>
+
+<property>
+  <name>fs.ftp.user.localhost</name>
+  <value>user</value>
+  <description>The username for connecting to FTP server running on localhost. 
+  This is required by FTPFileSystem</description>
+</property>
+
+<property>
+  <name>fs.ftp.password.localhost</name>
+  <value>password</value>
+  <description>The password for connecting to FTP server running on localhost.
+   This is required by FTPFileSystem</description>
+</property>
+
+<property>
+  <name>test.fs.s3n.name</name>
+  <value>s3n:///</value>
+  <description>The name of the s3n file system for testing.</description>
+</property>
+
+</configuration>

src/test/hadoop-site.xml

@@ -8,43 +8,5 @@
 <configuration>
 
 
-<property>
-  <name>hadoop.tmp.dir</name>
-  <value>build/test</value>
-  <description>A base for other temporary directories.</description>
-  <final>true</final>
-</property>
-
-<property>
-  <name>test.fs.s3.name</name>
-  <value>s3:///</value>
-  <description>The name of the s3 file system for testing.</description>
-</property>
-
-<property>
-  <name>fs.s3.block.size</name>
-  <value>128</value>
-  <description>Size of a block in bytes.</description>
-</property>
-
-<property>
-  <name>fs.ftp.user.localhost</name>
-  <value>user</value>
-  <description>The username for connecting to FTP server running on localhost. 
-  This is required by FTPFileSystem</description>
-</property>
-
-<property>
-  <name>fs.ftp.password.localhost</name>
-  <value>password</value>
-  <description>The password for connecting to FTP server running on localhost.
-   This is required by FTPFileSystem</description>
-</property>
-
-<property>
-  <name>test.fs.s3n.name</name>
-  <value>s3n:///</value>
-  <description>The name of the s3n file system for testing.</description>
-</property>
 
 </configuration>

src/test/hdfs-site.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+
+</configuration>

src/test/mapred-site.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+</configuration>

src/test/org/apache/hadoop/conf/TestConfiguration.java

@@ -224,7 +224,7 @@ public class TestConfiguration extends TestCase {
     conf.addResource(fileResource);
     
     String expectedOutput = 
-      "Configuration: hadoop-default.xml, hadoop-site.xml, " + 
+      "Configuration: core-default.xml, core-site.xml, " + 
       fileResource.toString();
     assertEquals(expectedOutput, conf.toString());
   }

src/test/org/apache/hadoop/hdfs/TestDFSShellGenericOptions.java

@@ -60,10 +60,10 @@ public class TestDFSShellGenericOptions extends TestCase {
   }
     
   private void testConfOption(String[] args, String namenode) {
-    // prepare configuration hadoop-site.xml
+    // prepare configuration hdfs-site.xml
     File configDir = new File(new File("build", "test"), "minidfs");
     assertTrue(configDir.mkdirs());
-    File siteFile = new File(configDir, "hadoop-site.xml");
+    File siteFile = new File(configDir, "hdfs-site.xml");
     PrintWriter pw;
     try {
       pw = new PrintWriter(siteFile);

src/test/org/apache/hadoop/mapred/MiniMRCluster.java

@@ -606,7 +606,7 @@ public class MiniMRCluster {
       stopJobTracker();
     } finally {
       File configDir = new File("build", "minimr");
-      File siteFile = new File(configDir, "hadoop-site.xml");
+      File siteFile = new File(configDir, "mapred-site.xml");
       siteFile.delete();
     }
   }