HDFS-14514. Actual read size of open file in encryption zone still larger than listing size even after enabling HDFS-11402 in Hadoop 2. Contributed by Siyao Meng, Stephen O'Donnell.

Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
Co-authored-by: Stephen O'Donnell <sodonnell@cloudera.com>
(cherry picked from commit cecba551aa3b03142830cfd58f548cb6b125ee7b)

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java

@@ -844,26 +844,18 @@ public class DFSInputStream extends FSInputStream
     @Override
     public int doRead(BlockReader blockReader, int off, int len)
         throws IOException {
-      int oldpos = buf.position();
-      int oldlimit = buf.limit();
-      boolean success = false;
-      try {
-        int ret = blockReader.read(buf);
-        success = true;
-        updateReadStatistics(readStatistics, ret, blockReader);
-        dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
-            ret);
-        if (ret == 0) {
-          DFSClient.LOG.warn("zero");
-        }
-        return ret;
-      } finally {
-        if (!success) {
-          // Reset to original state so that retries work correctly.
-          buf.position(oldpos);
-          buf.limit(oldlimit);
-        }
+      ByteBuffer tmpBuf = buf.duplicate();
+      tmpBuf.limit(tmpBuf.position() + len);
+      int nRead = blockReader.read(tmpBuf);
+      updateReadStatistics(readStatistics, nRead, blockReader);
+      dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
+          nRead);
+      if (nRead == 0) {
+        DFSClient.LOG.warn("zero");
+      } else if (nRead > 0) {
+        buf.position(buf.position() + nRead);
       }
+      return nRead;
     }
 
     @Override

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java

@@ -184,6 +184,8 @@ public class TestEncryptionZones {
     testRootDir = new File(testRoot).getAbsoluteFile();
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
         getKeyProviderURI());
+    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES,
+        true);
     conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
     // Lower the batch size for testing
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
@@ -1389,6 +1391,14 @@ public class TestEncryptionZones {
     assertEquals("Got unexpected ez path", zone.toString(),
         dfsAdmin.getEncryptionZoneForPath(snap1Zone).getPath().toString());
 
+    // Append the file
+    DFSTestUtil.appendFile(fs, zoneFile, len);
+    // Verify file content in the snapshot
+    final Path snapshottedZoneFile = new Path(
+        snap1.toString() + "/" + zone.getName() + "/" + zoneFile.getName());
+    assertEquals("Contents of snapshotted file have changed unexpectedly",
+        contents, DFSTestUtil.readFile(fs, snapshottedZoneFile));
+
     // Now delete the encryption zone, recreate the dir, and take another
     // snapshot
     fsWrapper.delete(zone, true);
@@ -1441,8 +1451,6 @@ public class TestEncryptionZones {
     assertEquals("Unexpected ez key", TEST_KEY2, listZone.getKeyName());
 
     // Verify contents of the snapshotted file
-    final Path snapshottedZoneFile = new Path(
-        snap1.toString() + "/" + zone.getName() + "/" + zoneFile.getName());
     assertEquals("Contents of snapshotted file have changed unexpectedly",
         contents, DFSTestUtil.readFile(fs, snapshottedZoneFile));