commit 2fbe620852d47a27f28efa17de1fd9378bd9170b

Author: Mahadev Konar <mahadev@cdev6022.inktomisearch.com>
Date:   Tue Oct 20 23:58:31 2009 +0000

    HADOOP-6097 from https://issues.apache.org/jira/secure/attachment/12422737/HADOOP-6097-0.20.patch
    
    +++ b/YAHOO-CHANGES.txt
    +    HADOOP-6097. Multiple bugs w/ Hadoop archives (mahadev)
    +


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security-patches@1077028 13f79535-47bb-0310-9956-ffa450edef68

src/core/org/apache/hadoop/fs/HarFileSystem.java

@@ -301,19 +301,8 @@ public class HarFileSystem extends FilterFileSystem {
     }
 
     URI tmpURI = fsPath.toUri();
-    fsPath = new Path(tmpURI.getPath());
     //change this to Har uri 
-    URI tmp = null;
-    try {
-      tmp = new URI(uri.getScheme(), harAuth, fsPath.toString(),
-                    tmpURI.getQuery(), tmpURI.getFragment());
-    } catch(URISyntaxException ue) {
-      LOG.error("Error in URI ", ue);
-    }
-    if (tmp != null) {
-      return new Path(tmp.toString());
-    }
-    return null;
+    return new Path(uri.getScheme(), harAuth, tmpURI.getPath());
   }
   
   /**
@@ -425,12 +414,13 @@ public class HarFileSystem extends FilterFileSystem {
       // do nothing just a read.
     }
     FSDataInputStream aIn = fs.open(archiveIndex);
-    LineReader aLin = new LineReader(aIn, getConf());
+    LineReader aLin;
     String retStr = null;
     // now start reading the real index file
-     read = 0;
     for (Store s: stores) {
+      read = 0;
       aIn.seek(s.begin);
+      aLin = new LineReader(aIn, getConf());
       while (read + s.begin < s.end) {
         int tmp = aLin.readLine(line);
         read += tmp;

src/test/org/apache/hadoop/fs/TestHarFileSystem.java

@@ -57,7 +57,7 @@ public class TestHarFileSystem extends TestCase {
   private MiniDFSCluster dfscluster;
   private MiniMRCluster mapred;
   private FileSystem fs;
-  private Path filea, fileb, filec;
+  private Path filea, fileb, filec, filed;
   private Path archivePath;
   
   protected void setUp() throws Exception {
@@ -69,6 +69,9 @@ public class TestHarFileSystem extends TestCase {
     filea = new Path(inputPath,"a");
     fileb = new Path(inputPath,"b");
     filec = new Path(inputPath,"c");
+    // check for har containing escape worthy characters
+    // in there name
+    filed = new Path(inputPath, "d%d");
     archivePath = new Path(fs.getHomeDirectory(), "tmp");
   }
   
@@ -121,7 +124,14 @@ public class TestHarFileSystem extends TestCase {
     out = fs.create(filec);
     out.write("c".getBytes());
     out.close();
+    out = fs.create(filed);
+    out.write("d".getBytes());
+    out.close();
     Configuration conf = mapred.createJobConf();
+    
+    // check to see if fs.har.impl.disable.cache is true
+    boolean archivecaching = conf.getBoolean("fs.har.impl.disable.cache", false);
+    assertTrue(archivecaching);
     HadoopArchives har = new HadoopArchives(conf);
     String[] args = new String[3];
     //check for destination not specfied
@@ -179,6 +189,7 @@ public class TestHarFileSystem extends TestCase {
     Path harFilea = new Path(harPath, "a");
     Path harFileb = new Path(harPath, "b");
     Path harFilec = new Path(harPath, "c");
+    Path harFiled = new Path(harPath, "d%d");
     FileSystem harFs = harFilea.getFileSystem(conf);
     FSDataInputStream fin = harFs.open(harFilea);
     byte[] b = new byte[4];
@@ -193,6 +204,11 @@ public class TestHarFileSystem extends TestCase {
     fin.read(b);
     fin.close();
     assertTrue("strings are equal ", (b[0] == "c".getBytes()[0]));
+    fin = harFs.open(harFiled);
+    fin.read(b);
+    fin.close();
+    assertTrue("strings are equal ", (b[0] == "d".getBytes()[0]));
+    
     // ok all files match 
     // run a map reduce job
     Path outdir = new Path(fs.getHomeDirectory(), "mapout"); 
@@ -213,11 +229,11 @@ public class TestHarFileSystem extends TestCase {
     FileStatus[] status = fs.globStatus(new Path(outdir, "part*"));
     Path reduceFile = status[0].getPath();
     FSDataInputStream reduceIn = fs.open(reduceFile);
-    b = new byte[6];
+    b = new byte[8];
     reduceIn.read(b);
-    //assuming all the 6 bytes were read.
+    //assuming all the 8 bytes were read.
     Text readTxt = new Text(b);
-    assertTrue("a\nb\nc\n".equals(readTxt.toString()));
+    assertTrue("a\nb\nc\nd\n".equals(readTxt.toString()));
     assertTrue("number of bytes left should be -1", reduceIn.read(b) == -1);
     reduceIn.close();
   }