Browse Source

HDFS-14535. The default 8KB buffer in requestFileDescriptors#BufferedOutputStream is causing lots of heap allocation in HBase when using short-circut read

(cherry picked from commit ea3b0a184405c3feca024a560807ea215b6858b9)
(cherry picked from commit 9b61ecfcbeeb1d3c320f775b43c68d3e39a271a3)
(cherry picked from commit b082628e5a39fc1bc9d6094beb9c253cc096c701)
(cherry picked from commit 2b9c4fccf4a8930bfff99ab4928b9b3b4980d3b2)
(cherry picked from commit 8b90c69357cf515661ec8a08e61ddd03e884b3ab)
(cherry picked from commit d14fd913f46fc8e794c39f4cc44d1674aa156ee0)
huzheng 6 years ago
parent
commit
daffd466b3

+ 2 - 1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java

@@ -86,6 +86,7 @@ import org.slf4j.LoggerFactory;
 @InterfaceAudience.Private
 public class BlockReaderFactory implements ShortCircuitReplicaCreator {
   static final Logger LOG = LoggerFactory.getLogger(BlockReaderFactory.class);
+  private static final int SMALL_BUFFER_SIZE = 512;
 
   public static class FailureInjector {
     public void injectRequestFileDescriptorsFailure() throws IOException {
@@ -589,7 +590,7 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
           Slot slot) throws IOException {
     ShortCircuitCache cache = clientContext.getShortCircuitCache();
     final DataOutputStream out =
-        new DataOutputStream(new BufferedOutputStream(peer.getOutputStream()));
+        new DataOutputStream(new BufferedOutputStream(peer.getOutputStream(), SMALL_BUFFER_SIZE));
     SlotId slotId = slot == null ? null : slot.getSlotId();
     new Sender(out).requestShortCircuitFds(block, token, slotId, 1,
         failureInjector.getSupportsReceiptVerification());