HADOOP-947. Merge -r 508624:508628 from trunk to 0.11 branch.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/branches/branch-0.11@508629 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -1,11 +1,14 @@
 Hadoop Change Log
 
 
-Branch 0.11 (unreleased)
+Release 0.11.2 - 2007-02-16
 
  1. HADOOP-1009.  Fix an infinite loop in the HDFS namenode.
     (Dhruba Borthakur via cutting) 
 
+ 2. HADOOP-947.  Disable in-memory merging during shuffle, as this is
+    causing data corruption.  (Devaraj Das via cutting)
+
 
 Release 0.11.1 - 2007-02-09
 

build.xml

@@ -9,7 +9,7 @@
  
   <property name="Name" value="Hadoop"/>
   <property name="name" value="hadoop"/>
-  <property name="version" value="0.11.2-dev"/>
+  <property name="version" value="0.11.3-dev"/>
   <property name="final.name" value="${name}-${version}"/>
   <property name="year" value="2006"/>
   <property name="libhdfs.version" value="1"/>

conf/hadoop-default.xml

@@ -132,7 +132,7 @@ creations/deletions), or "all".</description>
 
 <property>
   <name>fs.inmemory.size.mb</name>
-  <value>75</value>
+  <value>0</value>
   <description>The size of the in-memory filsystem instance in MB</description>
 </property>
 

docs/index.html

@@ -128,6 +128,9 @@ document.write("<text>Last Published:</text> " + document.lastModified);
 <a href="#News">News</a>
 <ul class="minitoc">
 <li>
+<a href="#16+February%2C+2007%3A+release+0.11.2+available">16 February, 2007: release 0.11.2 available</a>
+</li>
+<li>
 <a href="#9+February%2C+2007%3A+release+0.11.1+available">9 February, 2007: release 0.11.1 available</a>
 </li>
 <li>
@@ -214,133 +217,139 @@ document.write("<text>Last Published:</text> " + document.lastModified);
 <a name="N1000C"></a><a name="News"></a>
 <h2 class="h3">News</h2>
 <div class="section">
-<a name="N10012"></a><a name="9+February%2C+2007%3A+release+0.11.1+available"></a>
+<a name="N10012"></a><a name="16+February%2C+2007%3A+release+0.11.2+available"></a>
+<h3 class="h4">16 February, 2007: release 0.11.2 available</h3>
+<p>This fixes critical bugs in 0.11.1. For details see the <a href="http://tinyurl.com/2ukkt3">release notes</a>. The release can
+      be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
+      nearby mirror</a>.
+      </p>
+<a name="N10024"></a><a name="9+February%2C+2007%3A+release+0.11.1+available"></a>
 <h3 class="h4">9 February, 2007: release 0.11.1 available</h3>
 <p>This fixes bugs in 0.11.0. For details see the <a href="http://tinyurl.com/37c5a3">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N10024"></a><a name="2+February%2C+2007%3A+release+0.11.0+available"></a>
+<a name="N10036"></a><a name="2+February%2C+2007%3A+release+0.11.0+available"></a>
 <h3 class="h4">2 February, 2007: release 0.11.0 available</h3>
 <p>For details see the <a href="http://tinyurl.com/2yn8zs">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N10036"></a><a name="10+January%2C+2007%3A+release+0.10.1+available"></a>
+<a name="N10048"></a><a name="10+January%2C+2007%3A+release+0.10.1+available"></a>
 <h3 class="h4">10 January, 2007: release 0.10.1 available</h3>
 <p>This fixes critical bugs in 0.10.0.  For details see the <a href="http://tinyurl.com/ycpsms">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N10048"></a><a name="5+January%2C+2007%3A+release+0.10.0+available"></a>
+<a name="N1005A"></a><a name="5+January%2C+2007%3A+release+0.10.0+available"></a>
 <h3 class="h4">5 January, 2007: release 0.10.0 available</h3>
 <p>For details see the <a href="http://tinyurl.com/yapkxs">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N1005A"></a><a name="15+December%2C+2006%3A+release+0.9.2+available"></a>
+<a name="N1006C"></a><a name="15+December%2C+2006%3A+release+0.9.2+available"></a>
 <h3 class="h4">15 December, 2006: release 0.9.2 available</h3>
 <p>This fixes critical bugs in 0.9.1.  For details see the <a href="http://tinyurl.com/ya8lfd">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N1006C"></a><a name="6+December%2C+2006%3A+release+0.9.1+available"></a>
+<a name="N1007E"></a><a name="6+December%2C+2006%3A+release+0.9.1+available"></a>
 <h3 class="h4">6 December, 2006: release 0.9.1 available</h3>
 <p>This fixes critical bugs in 0.9.0.  For details see the <a href="http://tinyurl.com/y55d7p">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N1007E"></a><a name="1+December%2C+2006%3A+release+0.9.0+available"></a>
+<a name="N10090"></a><a name="1+December%2C+2006%3A+release+0.9.0+available"></a>
 <h3 class="h4">1 December, 2006: release 0.9.0 available</h3>
 <p>For details see the <a href="http://tinyurl.com/sdjhb">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N10090"></a><a name="3+November%2C+2006%3A+release+0.8.0+available"></a>
+<a name="N100A2"></a><a name="3+November%2C+2006%3A+release+0.8.0+available"></a>
 <h3 class="h4">3 November, 2006: release 0.8.0 available</h3>
 <p>For details see the <a href="http://tinyurl.com/ykqv6s">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N100A2"></a><a name="18+October%2C+2006%3A+release+0.7.2+available"></a>
+<a name="N100B4"></a><a name="18+October%2C+2006%3A+release+0.7.2+available"></a>
 <h3 class="h4">18 October, 2006: release 0.7.2 available</h3>
 <p>This fixes critical bugs in 0.7.1.  For details see the <a href="http://tinyurl.com/ygy6y7">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N100B4"></a><a name="11+October%2C+2006%3A+release+0.7.1+available"></a>
+<a name="N100C6"></a><a name="11+October%2C+2006%3A+release+0.7.1+available"></a>
 <h3 class="h4">11 October, 2006: release 0.7.1 available</h3>
 <p>This fixes critical bugs in 0.7.0.  For details see the <a href="http://tinyurl.com/p7qod">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N100C6"></a><a name="6+October%2C+2006%3A+release+0.7.0+available"></a>
+<a name="N100D8"></a><a name="6+October%2C+2006%3A+release+0.7.0+available"></a>
 <h3 class="h4">6 October, 2006: release 0.7.0 available</h3>
 <p>For details see the <a href="http://tinyurl.com/kvd9m">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N100D8"></a><a name="18+September%2C+2006%3A+release+0.6.2+available"></a>
+<a name="N100EA"></a><a name="18+September%2C+2006%3A+release+0.6.2+available"></a>
 <h3 class="h4">18 September, 2006: release 0.6.2 available</h3>
 <p>This fixes critical bugs in 0.6.1.  For details see the <a href="http://tinyurl.com/gyb56">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N100EA"></a><a name="13+September%2C+2006%3A+release+0.6.1+available"></a>
+<a name="N100FC"></a><a name="13+September%2C+2006%3A+release+0.6.1+available"></a>
 <h3 class="h4">13 September, 2006: release 0.6.1 available</h3>
 <p>This fixes critical bugs in 0.6.0.  For details see the <a href="http://tinyurl.com/lykp4">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N100FC"></a><a name="8+September%2C+2006%3A+release+0.6.0+available"></a>
+<a name="N1010E"></a><a name="8+September%2C+2006%3A+release+0.6.0+available"></a>
 <h3 class="h4">8 September, 2006: release 0.6.0 available</h3>
 <p>For details see the <a href="http://tinyurl.com/r3zoj">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N1010E"></a><a name="4+August%2C+2006%3A+release+0.5.0+available"></a>
+<a name="N10120"></a><a name="4+August%2C+2006%3A+release+0.5.0+available"></a>
 <h3 class="h4">4 August, 2006: release 0.5.0 available</h3>
 <p>For details see the <a href="http://tinyurl.com/pnml2">release notes</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N10120"></a><a name="28+June%2C+2006%3A+release+0.4.0+available"></a>
+<a name="N10132"></a><a name="28+June%2C+2006%3A+release+0.4.0+available"></a>
 <h3 class="h4">28 June, 2006: release 0.4.0 available</h3>
 <p>For details see the <a href="http://tinyurl.com/o35b6">change log</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N10132"></a><a name="9+June%2C+2006%3A+release+0.3.2+available"></a>
+<a name="N10144"></a><a name="9+June%2C+2006%3A+release+0.3.2+available"></a>
 <h3 class="h4">9 June, 2006: release 0.3.2 available</h3>
 <p>This is a bugfix release.  For details see the <a href="http://tinyurl.com/k9g5c">change log</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N10144"></a><a name="8+June%2C+2006%3A+FAQ+added+to+Wiki"></a>
+<a name="N10156"></a><a name="8+June%2C+2006%3A+FAQ+added+to+Wiki"></a>
 <h3 class="h4">8 June, 2006: FAQ added to Wiki</h3>
 <p>Hadoop now has a <a href="http://wiki.apache.org/lucene-hadoop/FAQ">FAQ</a>.  Please
       help make this more complete!
       </p>
-<a name="N10152"></a><a name="5+June%2C+2006%3A+release+0.3.1+available"></a>
+<a name="N10164"></a><a name="5+June%2C+2006%3A+release+0.3.1+available"></a>
 <h3 class="h4">5 June, 2006: release 0.3.1 available</h3>
 <p>This is a bugfix release.  For details see the <a href="http://tinyurl.com/l6on4">change log</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N10164"></a><a name="2+June%2C+2006%3A+release+0.3.0+available"></a>
+<a name="N10176"></a><a name="2+June%2C+2006%3A+release+0.3.0+available"></a>
 <h3 class="h4">2 June, 2006: release 0.3.0 available</h3>
 <p>This includes many fixes, improving performance, scalability
       and reliability and adding new features.  For details see the <a href="http://tinyurl.com/rq3f7">change log</a>. The release can
       be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N10176"></a><a name="12+May%2C+2006%3A+release+0.2.1+available"></a>
+<a name="N10188"></a><a name="12+May%2C+2006%3A+release+0.2.1+available"></a>
 <h3 class="h4">12 May, 2006: release 0.2.1 available</h3>
 <p>This fixes a few bugs in release 0.2.0, listed in the <a href="http://tinyurl.com/rnnvz">change log</a>. The
       release can be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N10188"></a><a name="5+May%2C+2006%3A+release+0.2.0+available"></a>
+<a name="N1019A"></a><a name="5+May%2C+2006%3A+release+0.2.0+available"></a>
 <h3 class="h4">5 May, 2006: release 0.2.0 available</h3>
 <p>We are now aiming for monthly releases.  There have been many
       bug fixes and improvements in the past month.  MapReduce and DFS
@@ -349,24 +358,24 @@ document.write("<text>Last Published:</text> " + document.lastModified);
       details. The release can be obtained from <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
       nearby mirror</a>.
       </p>
-<a name="N1019A"></a><a name="2+April%2C+2006%3A+release+0.1.0+available"></a>
+<a name="N101AC"></a><a name="2+April%2C+2006%3A+release+0.1.0+available"></a>
 <h3 class="h4">2 April, 2006: release 0.1.0 available</h3>
 <p>This is the first Hadoop release.  The release is available
       <a href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/">
       here</a>.</p>
-<a name="N101A8"></a><a name="6+February%2C+2006%3A+nightly+builds"></a>
+<a name="N101BA"></a><a name="6+February%2C+2006%3A+nightly+builds"></a>
 <h3 class="h4">6 February, 2006: nightly builds</h3>
 <p>Hadoop now has nightly builds.  This automatically creates a
       <a href="http://cvs.apache.org/dist/lucene/hadoop/nightly/">downloadable version of Hadoop every
       night</a>.  All unit tests must pass, or a message is sent to
       the developers mailing list and no new version is created.  This
       also updates the <a href="api/index.html">javadoc</a>.</p>
-<a name="N101BA"></a><a name="3+February%2C+2006%3A+Hadoop+code+moved+out+of+Nutch"></a>
+<a name="N101CC"></a><a name="3+February%2C+2006%3A+Hadoop+code+moved+out+of+Nutch"></a>
 <h3 class="h4">3 February, 2006: Hadoop code moved out of Nutch</h3>
 <p>The Hadoop code has now been moved into its own Subversion
       tree, renamed into packages under <span class="codefrag">org.apache.hadoop</span>.
       All unit tests pass, but little else has yet been tested.</p>
-<a name="N101C7"></a><a name="30+March%2C+2006%3A+Hadoop+project+approved"></a>
+<a name="N101D9"></a><a name="30+March%2C+2006%3A+Hadoop+project+approved"></a>
 <h3 class="h4">30 March, 2006: Hadoop project approved</h3>
 <p>The Lucene PMC has elected to split the Nutch MapReduce and
       distributed filesytem code into a new project named Hadoop.</p>

src/docs/src/documentation/content/xdocs/index.xml

@@ -14,6 +14,15 @@
     <section>
       <title>News</title>
 
+      <section>
+      <title>16 February, 2007: release 0.11.2 available</title>
+      <p>This fixes critical bugs in 0.11.1. For details see the <a
+      href="http://tinyurl.com/2ukkt3">release notes</a>. The release can
+      be obtained from <a
+      href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
+      nearby mirror</a>.
+      </p> </section>
+
       <section>
       <title>9 February, 2007: release 0.11.1 available</title>
       <p>This fixes bugs in 0.11.0. For details see the <a

src/java/org/apache/hadoop/fs/InMemoryFileSystem.java

@@ -386,7 +386,9 @@ public class InMemoryFileSystem extends FileSystem {
   }
   
   public float getPercentUsed() {
-    return (float)totalUsed/fsSize;
+    if (fsSize > 0)
+      return (float)totalUsed/fsSize;
+    else return 0.1f;
   }
  
   private boolean canFitInMemory(int size) {

src/java/org/apache/hadoop/mapred/MapOutputLocation.java

@@ -204,8 +204,9 @@ class MapOutputLocation implements Writable, MRConstants {
       int inMemFSSize = inMemFileSys.getFSSize();
       int checksumLength = inMemFileSys.getChecksumFileLength(length);
         
-      boolean createInMem = 
-        (((float)(length + checksumLength) / inMemFSSize <= 
+      boolean createInMem = false; 
+      if (inMemFSSize > 0)  
+        createInMem = (((float)(length + checksumLength) / inMemFSSize <= 
         MAX_INMEM_FILESIZE_FRACTION) && 
         inMemFileSys.reserveSpaceWithCheckSum(localFilename, length));