Home Explore Help
Register Sign In
apache
/
hadoop
mirror of https://github.com/apache/hadoop.git
2
0
Fork 1
Files Issues 0 Wiki
Browse Source

First version that passes unit tests.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@374740 13f79535-47bb-0310-9956-ffa450edef68
Doug Cutting 19 years ago
parent
1e3ec8df8a
commit
0bafd717da
12 changed files with 63 additions and 42 deletions
Split View Show Diff Stats
  1. 1 4
      build.xml
  2. 24 0
      conf/configuration.xsl
  3. 10 10
      conf/hadoop-default.xml
  4. 12 12
      src/java/org/apache/hadoop/conf/Configuration.java
  5. 1 1
      src/java/org/apache/hadoop/dfs/Block.java
  6. 1 1
      src/java/org/apache/hadoop/dfs/DFSClient.java
  7. 1 1
      src/java/org/apache/hadoop/dfs/FSNamesystem.java
  8. 1 1
      src/java/org/apache/hadoop/dfs/NameNode.java
  9. 5 5
      src/java/org/apache/hadoop/fs/FileSystem.java
  10. 1 1
      src/java/org/apache/hadoop/mapred/JobConf.java
  11. 5 5
      src/test/org/apache/hadoop/dfs/TestDFS.java
  12. 1 1
      src/test/org/apache/hadoop/mapred/MapredLoadTest.java

+ 1 - 4
build.xml
View File 

@@ -147,9 +147,6 @@
 
     <delete dir="${test.build.data}"/>
 
     <mkdir dir="${test.build.data}"/>
 
 
-    <copy file="${test.src.dir}/hadoop-site.xml"
 
-          todir="${test.build.classes}"/>
 
-
 
     <junit printsummary="yes" haltonfailure="no" fork="yes" dir="${basedir}"
 
       errorProperty="tests.failed" failureProperty="tests.failed">
 
       <sysproperty key="test.build.data" value="${test.build.data}"/>
 
@@ -195,7 +192,7 @@
 
 	
   <target name="default-doc">
 
     <style basedir="${conf.dir}" destdir="${docs.dir}"
 
-           includes="hadoop-default.xml" style="conf/hadoop-conf.xsl"/>
 
+           includes="hadoop-default.xml" style="conf/configuration.xsl"/>
 
   </target>
 
 
   <!-- ================================================================== -->

+ 24 - 0
conf/configuration.xsl
View File 

@@ -0,0 +1,24 @@
 
+<?xml version="1.0"?>
 
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
 
+<xsl:output method="html"/>
 
+<xsl:template match="nutch-conf">
 
+<html>
 
+<body>
 
+<table border="1">
 
+<tr>
 
+ <td>name</td>
 
+ <td>value</td>
 
+ <td>description</td>
 
+</tr>
 
+<xsl:for-each select="property">
 
+<tr>
 
+  <td><xsl:value-of select="name"/></td>
 
+  <td><xsl:value-of select="value"/></td>
 
+  <td><xsl:value-of select="description"/></td>
 
+</tr>
 
+</xsl:for-each>
 
+</table>
 
+</body>
 
+</html>
 
+</xsl:template>
 
+</xsl:stylesheet>

+ 10 - 10
conf/hadoop-default.xml
View File 

@@ -1,11 +1,11 @@
 
 <?xml version="1.0"?>
 
-<?xml-stylesheet type="text/xsl" href="nutch-conf.xsl"?>
 
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 
 
 <!-- Do not modify this file directly.  Instead, copy entries that you -->
 
-<!-- wish to modify from this file into nutch-site.xml and change them -->
 
-<!-- there.  If nutch-site.xml does not already exist, create it.      -->
 
+<!-- wish to modify from this file into hadoop-site.xml and change them -->
 
+<!-- there.  If hadoop-site.xml does not already exist, create it.      -->
 
 
-<nutch-conf>
 
+<configuration>
 
 
 <!-- file properties -->
 
 
@@ -97,14 +97,14 @@
 
 
 <property>
 
   <name>dfs.name.dir</name>
 
-  <value>/tmp/nutch/dfs/name</value>
 
+  <value>/tmp/hadoop/dfs/name</value>
 
   <description>Determines where on the local filesystem the DFS name node
 
       should store the name table.</description>
 
 </property>
 
 
 <property>
 
   <name>dfs.data.dir</name>
 
-  <value>/tmp/nutch/dfs/data</value>
 
+  <value>/tmp/hadoop/dfs/data</value>
 
   <description>Determines where on the local filesystem an DFS data node
 
   should store its blocks.  If this is a comma- or space-delimited
 
   list of directories, then data will be stored in all named
 
@@ -155,7 +155,7 @@
 
 
 <property>
 
   <name>mapred.local.dir</name>
 
-  <value>/tmp/nutch/mapred/local</value>
 
+  <value>/tmp/hadoop/mapred/local</value>
 
   <description>The local directory where MapReduce stores intermediate
 
   data files.  May be a space- or comma- separated list of
 
   directories on different devices in order to spread disk i/o.
 
@@ -164,14 +164,14 @@
 
 
 <property>
 
   <name>mapred.system.dir</name>
 
-  <value>/tmp/nutch/mapred/system</value>
 
+  <value>/tmp/hadoop/mapred/system</value>
 
   <description>The shared directory where MapReduce stores control files.
 
   </description>
 
 </property>
 
 
 <property>
 
   <name>mapred.temp.dir</name>
 
-  <value>/tmp/nutch/mapred/temp</value>
 
+  <value>/tmp/hadoop/mapred/temp</value>
 
   <description>A shared directory for temporary files.
 
   </description>
 
 </property>
 
@@ -234,4 +234,4 @@
 
   <description>Defines the timeout for IPC calls in milliseconds.</description>
 
 </property>
 
 
-</nutch-conf>
 
+</configuration>

+ 12 - 12
src/java/org/apache/hadoop/conf/Configuration.java
View File 

@@ -35,10 +35,10 @@ import org.apache.hadoop.util.LogFormatter;
 
  * <p>An ordered list of configuration parameter files with
 
  * default and always-overrides site parameters.
 
  * <p>Default values for all parameters are specified in a file named
 
- * <tt>nutch-default.xml</tt> located on the classpath.  Overrides for these
 
- * defaults should be in an optional file named <tt>nutch-site.xml</tt>, also
 
+ * <tt>hadoop-default.xml</tt> located on the classpath.  Overrides for these
 
+ * defaults should be in an optional file named <tt>hadoop-site.xml</tt>, also
 
  * located on the classpath.  Typically these files reside in the
 
- * <tt>conf/</tt> subdirectory at the top-level of a Nutch installation.
 
+ * <tt>conf/</tt> subdirectory at the top-level of a Hadoop installation.
 
  * <p>The resource files are read upon first access of values (set, get,
 
  * or write) after {@link #addConfResource(String)} or
 
  * {@link #addConfResource(File)}.
 
@@ -54,8 +54,8 @@ public class Configuration {
 
 
   /** A new configuration. */
 
   public Configuration() {
 
-    resourceNames.add("nutch-default.xml");
 
-    resourceNames.add("nutch-site.xml");
 
+    resourceNames.add("hadoop-default.xml");
 
+    resourceNames.add("hadoop-site.xml");
 
   }
 
 
   /** A new configuration with the same settings cloned from another. */
 
@@ -67,16 +67,16 @@ public class Configuration {
 
 
   /** Adds a resource name to the chain of resources read.  Such resources are
 
    * located on the CLASSPATH.  The first resource is always
 
-   * <tt>nutch-default.xml</tt>, and the last is always
 
-   * <tt>nutch-site.xml</tt>.  New resources are inserted between these, so
 
+   * <tt>hadoop-default.xml</tt>, and the last is always
 
+   * <tt>hadoop-site.xml</tt>.  New resources are inserted between these, so
 
    * they can override defaults, but not site-specifics. */
 
   public synchronized void addConfResource(String name) {
 
     addConfResourceInternal(name);
 
   }
 
 
   /** Adds a file to the chain of resources read.  The first resource is always
 
-   * <tt>nutch-default.xml</tt>, and the last is always
 
-   * <tt>nutch-site.xml</tt>.  New resources are inserted between these, so
 
+   * <tt>hadoop-default.xml</tt>, and the last is always
 
+   * <tt>hadoop-site.xml</tt>.  New resources are inserted between these, so
 
    * they can override defaults, but not site-specifics. */
 
   public synchronized void addConfResource(File file) {
 
     addConfResourceInternal(file);
 
@@ -344,8 +344,8 @@ public class Configuration {
 
       }
 
 
       Element root = doc.getDocumentElement();
 
-      if (!"nutch-conf".equals(root.getTagName()))
 
-        LOG.severe("bad conf file: top-level element not <nutch-conf>");
 
+      if (!"configuration".equals(root.getTagName()))
 
+        LOG.severe("bad conf file: top-level element not <configuration>");
 
       NodeList props = root.getChildNodes();
 
       for (int i = 0; i < props.getLength(); i++) {
 
         Node propNode = props.item(i);
 
@@ -384,7 +384,7 @@ public class Configuration {
 
     try {
 
       Document doc =
 
         DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
 
-      Element conf = doc.createElement("nutch-conf");
 
+      Element conf = doc.createElement("configuration");
 
       doc.appendChild(conf);
 
       conf.appendChild(doc.createTextNode("\n"));
 
       for (Enumeration e = properties.keys(); e.hasMoreElements();) {

+ 1 - 1
src/java/org/apache/hadoop/dfs/Block.java
View File 

@@ -21,7 +21,7 @@ import java.io.*;
 
 import java.util.*;
 
 
 /**************************************************
 
- * A Block is a Nutch FS primitive, identified by a 
+ * A Block is a Hadoop FS primitive, identified by a 
  * long.
 
  *
 
  * @author Mike Cafarella

+ 1 - 1
src/java/org/apache/hadoop/dfs/DFSClient.java
View File 

@@ -27,7 +27,7 @@ import java.util.*;
 
 import java.util.logging.*;
 
 
 /********************************************************
 
- * DFSClient can connect to a Nutch Filesystem and perform basic file tasks.
 
+ * DFSClient can connect to a Hadoop Filesystem and perform basic file tasks.
 
  * Connects to a namenode daemon.
 
  * @author Mike Cafarella, Tessa MacDuff
 
  ********************************************************/

+ 1 - 1
src/java/org/apache/hadoop/dfs/FSNamesystem.java
View File 

@@ -191,7 +191,7 @@ public class FSNamesystem implements FSConstants {
 
 
     /////////////////////////////////////////////////////////
 
     //
 
-    // These methods are called by NutchFS clients
 
+    // These methods are called by HadoopFS clients
 
     //
 
     /////////////////////////////////////////////////////////
 
     /**

+ 1 - 1
src/java/org/apache/hadoop/dfs/NameNode.java
View File 

@@ -51,7 +51,7 @@ public class NameNode implements ClientProtocol, DatanodeProtocol, FSConstants {
 
      */
 
     public NameNode(Configuration conf) throws IOException {
 
         this(new File(conf.get("dfs.name.dir",
 
-                                          "/tmp/nutch/dfs/name")),
 
+                                          "/tmp/hadoop/dfs/name")),
 
              DataNode.createSocketAddr
 
              (conf.get("fs.default.name", "local")).getPort(), conf);
 
     }

+ 5 - 5
src/java/org/apache/hadoop/fs/FileSystem.java
View File 

@@ -27,15 +27,15 @@ import org.apache.hadoop.util.LogFormatter;
 
 /****************************************************************
 
  * An abstract base class for a fairly simple
 
  * distributed file system.
 
- * A Nutch installation might consist
 
+ * A Hadoop installation might consist
 
  * of multiple machines, which should swap files transparently.
 
- * This interface allows other Nutch systems to find and place
 
- * files into the distributed Nutch-controlled file world.
 
+ * This interface allows other Hadoop systems to find and place
 
+ * files into the distributed Hadoop-controlled file world.
 
  * <p>
 
- * A local implementation exists for testing and for small Nutch instances.
 
+ * A local implementation exists for testing and for small Hadoop instances.
 
  * <p>
 
  * The standard job of FileSystem is to take the location-
 
- * independent NutchFile objects, and resolve them using local
 
+ * independent HadoopFile objects, and resolve them using local
 
  * knowledge and local instances of ShareGroup.
 
  * <p>
 
  * The local implementation is {@link LocalFileSystem} and distributed

+ 1 - 1
src/java/org/apache/hadoop/mapred/JobConf.java
View File 

@@ -90,7 +90,7 @@ public class JobConf extends Configuration {
 
 
   public File getSystemDir() {
 
     return new File(get("mapred.system.dir",
 
-                                        "/tmp/nutch/mapred/system"));
 
+                                        "/tmp/hadoop/mapred/system"));
 
   }
 
 
   public String[] getLocalDirs() throws IOException {

+ 5 - 5
src/test/org/apache/hadoop/dfs/TestDFS.java
View File 

@@ -65,7 +65,7 @@ import java.lang.reflect.InvocationTargetException;
 
  * <p>Bring the namenode down and restart it to verify that datanodes reconnect.
 
  * <p>
 
  * <p>For a another approach to filesystem testing, see the high level
 
- * (NutchFS level) test {@link org.apache.hadoop.fs.TestFileSystem}.
 
+ * (HadoopFS level) test {@link org.apache.hadoop.fs.TestFileSystem}.
 
  * @author Paul Baclace
 
  */
 
 public class TestDFS extends TestCase implements FSConstants {
 
@@ -199,8 +199,8 @@ public class TestDFS extends TestCase implements FSConstants {
 
     // downsize for testing (just to save resources)
 
     conf.setInt("dfs.namenode.handler.count", 3);
 
     if (false) { //  use MersenneTwister, if present
 
-      conf.set("nutch.random.class",
 
-                          "org.apache.nutch.util.MersenneTwister");
 
+      conf.set("hadoop.random.class",
 
+                          "org.apache.hadoop.util.MersenneTwister");
 
     }
 
     conf.setLong("dfs.blockreport.intervalMsec", 50*1000L);
 
     conf.setLong("dfs.datanode.startupMsec", 15*1000L);
 
@@ -425,7 +425,7 @@ public class TestDFS extends TestCase implements FSConstants {
 
   /**
 
    * Make a data generator.
 
    * Allows optional use of high quality PRNG by setting property
 
-   * nutch.random.class to the full class path of a subclass of
 
+   * hadoop.random.class to the full class path of a subclass of
 
    * java.util.Random such as "...util.MersenneTwister".
 
    * The property test.dfs.random.seed can supply a seed for reproducible
 
    * testing (a default is set here if property is not set.)
 
@@ -436,7 +436,7 @@ public class TestDFS extends TestCase implements FSConstants {
 
       if (randomDataGeneratorCtor == null) {
 
         // lazy init
 
         String rndDataGenClassname =
 
-            conf.get("nutch.random.class", "java.util.Random");
 
+            conf.get("hadoop.random.class", "java.util.Random");
 
         Class clazz = Class.forName(rndDataGenClassname);
 
         randomDataGeneratorCtor = clazz.getConstructor(new Class[]{Long.TYPE});
 
       }

+ 1 - 1
src/test/org/apache/hadoop/mapred/MapredLoadTest.java
View File 

@@ -26,7 +26,7 @@ import java.math.*;
 
 
 /**********************************************************
 
  * MapredLoadTest generates a bunch of work that exercises
 
- * a Nutch Map-Reduce system (and DFS, too).  It goes through
 
+ * a Hadoop Map-Reduce system (and DFS, too).  It goes through
 
  * the following steps:
 
  *
 
  * 1) Take inputs 'range' and 'counts'.