MAPREDUCE-4133. MR over viewfs is broken (John George via bobby)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1329403 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -395,6 +395,7 @@ Release 0.23.3 - UNRELEASED
     MAPREDUCE-4190. Improve web UI for task attempts userlog link (Tom Graves
     via bobby)
 
+    MAPREDUCE-4133. MR over viewfs is broken (John George via bobby)
 
 Release 0.23.2 - UNRELEASED
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java

@@ -827,7 +827,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
         || (sysMaxReduces == 1);
 
     long sysMaxBytes = conf.getLong(MRJobConfig.JOB_UBERTASK_MAXBYTES,
-        fs.getDefaultBlockSize()); // FIXME: this is wrong; get FS from
+        fs.getDefaultBlockSize(this.remoteJobSubmitDir)); // FIXME: this is wrong; get FS from
                                    // [File?]InputFormat and default block size
                                    // from that
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java

@@ -128,7 +128,7 @@ public class DistributedFSCheck extends TestCase {
     if (rootStatus.isFile()) {
       nrFiles++;
       // For a regular file generate <fName,offset> pairs
-      long blockSize = fs.getDefaultBlockSize();
+      long blockSize = fs.getDefaultBlockSize(rootFile);
       long fileLength = rootStatus.getLen();
       for(long offset = 0; offset < fileLength; offset += blockSize)
         writer.append(new Text(rootFile.toString()), new LongWritable(offset));
@@ -160,15 +160,16 @@ public class DistributedFSCheck extends TestCase {
                        ) throws IOException {
       // open file
       FSDataInputStream in = null;
+      Path p = new Path(name);
       try {
-        in = fs.open(new Path(name));
+        in = fs.open(p);
       } catch(IOException e) {
         return name + "@(missing)";
       }
       in.seek(offset);
       long actualSize = 0;
       try {
-        long blockSize = fs.getDefaultBlockSize();
+        long blockSize = fs.getDefaultBlockSize(p);
         reporter.setStatus("reading " + name + "@" + 
                            offset + "/" + blockSize);
         for( int curSize = bufferSize; 

hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java

@@ -287,7 +287,7 @@ public class JobHistory {
     FSDataOutputStream out = logDirFs.create(logFile, 
         new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
         true, defaultBufferSize, 
-        logDirFs.getDefaultReplication(), 
+        logDirFs.getDefaultReplication(logFile), 
         jobHistoryBlockSize, null);
   
     EventWriter writer = new EventWriter(out);
@@ -306,8 +306,8 @@ public class JobHistory {
           jobFileOut = logDirFs.create(logDirConfPath,
               new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
               true, defaultBufferSize,
-              logDirFs.getDefaultReplication(),
-              logDirFs.getDefaultBlockSize(), null);
+              logDirFs.getDefaultReplication(logDirConfPath),
+              logDirFs.getDefaultBlockSize(logDirConfPath), null);
           jobConf.writeXml(jobFileOut);
           jobFileOut.close();
         }