MAPREDUCE-6252. JobHistoryServer should not fail when encountering a

missing directory. Contributed by Craig Welch.

hadoop-mapreduce-project/CHANGES.txt

@@ -349,6 +349,9 @@ Release 2.8.0 - UNRELEASED
     MAPREDUCE-6333. TestEvents,TestAMWebServicesTasks,TestAppController are
     broken due to MAPREDUCE-6297. (Siqi Li via gera)
 
+    MAPREDUCE-6252. JobHistoryServer should not fail when encountering a 
+    missing directory. (Craig Welch via devaraj)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java

@@ -740,17 +740,22 @@ public class HistoryFileManager extends AbstractService {
     }
   }
 
-  private static List<FileStatus> scanDirectory(Path path, FileContext fc,
+  @VisibleForTesting
+  protected static List<FileStatus> scanDirectory(Path path, FileContext fc,
       PathFilter pathFilter) throws IOException {
     path = fc.makeQualified(path);
     List<FileStatus> jhStatusList = new ArrayList<FileStatus>();
-    RemoteIterator<FileStatus> fileStatusIter = fc.listStatus(path);
-    while (fileStatusIter.hasNext()) {
-      FileStatus fileStatus = fileStatusIter.next();
-      Path filePath = fileStatus.getPath();
-      if (fileStatus.isFile() && pathFilter.accept(filePath)) {
-        jhStatusList.add(fileStatus);
+    try {
+      RemoteIterator<FileStatus> fileStatusIter = fc.listStatus(path);
+      while (fileStatusIter.hasNext()) {
+        FileStatus fileStatus = fileStatusIter.next();
+        Path filePath = fileStatus.getPath();
+        if (fileStatus.isFile() && pathFilter.accept(filePath)) {
+          jhStatusList.add(fileStatus);
+        }
       }
+    } catch (FileNotFoundException fe) {
+      LOG.error("Error while scanning directory " + path, fe);
     }
     return jhStatusList;
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java

@@ -21,13 +21,17 @@ package org.apache.hadoop.mapreduce.v2.hs;
 
 import java.io.File;
 import java.io.FileOutputStream;
+import java.io.FileNotFoundException;
 import java.util.UUID;
+import java.util.List;
 
 import org.junit.Assert;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -45,6 +49,8 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestName;
 
+import static org.mockito.Mockito.*;
+
 public class TestHistoryFileManager {
   private static MiniDFSCluster dfsCluster = null;
   private static MiniDFSCluster dfsCluster2 = null;
@@ -199,4 +205,20 @@ public class TestHistoryFileManager {
     testCreateHistoryDirs(dfsCluster.getConfiguration(0), clock);
   }
 
+  @Test
+  public void testScanDirectory() throws Exception {
+
+    Path p = new Path("any");
+    FileContext fc = mock(FileContext.class);
+    when(fc.makeQualified(p)).thenReturn(p);
+    when(fc.listStatus(p)).thenThrow(new FileNotFoundException());
+
+    List<FileStatus> lfs = HistoryFileManager.scanDirectory(p, fc, null);
+
+    //primarily, succcess is that an exception was not thrown.  Also nice to
+    //check this
+    Assert.assertNotNull(lfs);
+
+  }
+
 }