HADOOP-19389: Optimize shell -text command I/O with multi-byte read. (#7291)

Reviewed-by: Ashutosh Gupta <ashutosh.gupta@st.niituniversity.in>
Signed-off-by: Steve Loughran <stevel@apache.org>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java

@@ -215,12 +215,12 @@ class Display extends FsCommand {
   }
 
   protected class TextRecordInputStream extends InputStream {
-    SequenceFile.Reader r;
-    Object key;
-    Object val;
+    private final SequenceFile.Reader r;
+    private Object key;
+    private Object val;
 
-    DataInputBuffer inbuf;
-    DataOutputBuffer outbuf;
+    private final DataInputBuffer inbuf;
+    private final DataOutputBuffer outbuf;
 
     public TextRecordInputStream(FileStatus f) throws IOException {
       final Path fpath = f.getPath();
@@ -237,30 +237,67 @@ class Display extends FsCommand {
     public int read() throws IOException {
       int ret;
       if (null == inbuf || -1 == (ret = inbuf.read())) {
-        key = r.next(key);
-        if (key == null) {
-          return -1;
+        if (!readNextFromSequenceFile()) {
+          ret = -1;
         } else {
-          val = r.getCurrentValue(val);
+          ret = inbuf.read();
         }
-        byte[] tmp = key.toString().getBytes(StandardCharsets.UTF_8);
-        outbuf.write(tmp, 0, tmp.length);
-        outbuf.write('\t');
-        tmp = val.toString().getBytes(StandardCharsets.UTF_8);
-        outbuf.write(tmp, 0, tmp.length);
-        outbuf.write('\n');
-        inbuf.reset(outbuf.getData(), outbuf.getLength());
-        outbuf.reset();
-        ret = inbuf.read();
       }
       return ret;
     }
 
+    @Override
+    public int read(byte[] dest, int destPos, int destLen) throws IOException {
+      validateInputStreamReadArguments(dest, destPos, destLen);
+
+      if (destLen == 0) {
+        return 0;
+      }
+
+      int bytesRead = 0;
+      while (destLen > 0) {
+        // Attempt to copy buffered data.
+        int copyLen = inbuf.read(dest, destPos, destLen);
+        if (-1 == copyLen) {
+          // There was no buffered data.
+          if (!readNextFromSequenceFile()) {
+            // There is also no data remaining in the file.
+            break;
+          }
+          // Reattempt copy now that we have buffered data.
+          copyLen = inbuf.read(dest, destPos, destLen);
+        }
+        bytesRead += copyLen;
+        destPos += copyLen;
+        destLen -= copyLen;
+      }
+
+      return bytesRead > 0 ? bytesRead : -1;
+    }
+
     @Override
     public void close() throws IOException {
       r.close();
       super.close();
     }
+
+    private boolean readNextFromSequenceFile() throws IOException {
+      key = r.next(key);
+      if (key == null) {
+        return false;
+      } else {
+        val = r.getCurrentValue(val);
+      }
+      byte[] tmp = key.toString().getBytes(StandardCharsets.UTF_8);
+      outbuf.write(tmp, 0, tmp.length);
+      outbuf.write('\t');
+      tmp = val.toString().getBytes(StandardCharsets.UTF_8);
+      outbuf.write(tmp, 0, tmp.length);
+      outbuf.write('\n');
+      inbuf.reset(outbuf.getData(), outbuf.getLength());
+      outbuf.reset();
+      return true;
+    }
   }
 
   /**
@@ -270,10 +307,11 @@ class Display extends FsCommand {
   protected static class AvroFileInputStream extends InputStream {
     private int pos;
     private byte[] buffer;
-    private ByteArrayOutputStream output;
-    private FileReader<?> fileReader;
-    private DatumWriter<Object> writer;
-    private JsonEncoder encoder;
+    private final ByteArrayOutputStream output;
+    private final FileReader<?> fileReader;
+    private final DatumWriter<Object> writer;
+    private final JsonEncoder encoder;
+    private final byte[] finalSeparator;
 
     public AvroFileInputStream(FileStatus status) throws IOException {
       pos = 0;
@@ -286,6 +324,7 @@ class Display extends FsCommand {
       writer = new GenericDatumWriter<Object>(schema);
       output = new ByteArrayOutputStream();
       encoder = EncoderFactory.get().jsonEncoder(schema, output);
+      finalSeparator = System.getProperty("line.separator").getBytes(StandardCharsets.UTF_8);
     }
 
     /**
@@ -293,24 +332,88 @@ class Display extends FsCommand {
      */
     @Override
     public int read() throws IOException {
+      if (buffer == null) {
+        return -1;
+      }
+
       if (pos < buffer.length) {
         return buffer[pos++];
       }
+
       if (!fileReader.hasNext()) {
+        // Unset buffer to signal EOF on future calls.
+        buffer = null;
         return -1;
       }
+
       writer.write(fileReader.next(), encoder);
       encoder.flush();
+
       if (!fileReader.hasNext()) {
-        // Write a new line after the last Avro record.
-        output.write(System.getProperty("line.separator")
-                         .getBytes(StandardCharsets.UTF_8));
-        output.flush();
+        if (buffer.length > 0) {
+          // Write a new line after the last Avro record.
+          output.write(finalSeparator);
+          output.flush();
+        }
       }
+
+      swapBuffer();
+      return read();
+    }
+
+    @Override
+    public int read(byte[] dest, int destPos, int destLen) throws IOException {
+      validateInputStreamReadArguments(dest, destPos, destLen);
+
+      if (destLen == 0) {
+        return 0;
+      }
+
+      if (buffer == null) {
+        return -1;
+      }
+
+      int bytesRead = 0;
+      while (destLen > 0 && buffer != null) {
+        if (pos < buffer.length) {
+          // We have buffered data available, either from the Avro file or the final separator.
+          int copyLen = Math.min(buffer.length - pos, destLen);
+          System.arraycopy(buffer, pos, dest, destPos, copyLen);
+          pos += copyLen;
+          bytesRead += copyLen;
+          destPos += copyLen;
+          destLen -= copyLen;
+        } else if (buffer == finalSeparator) {
+          // There is no buffered data, and the last buffer processed was the final separator.
+          // Unset buffer to signal EOF on future calls.
+          buffer = null;
+        } else if (!fileReader.hasNext()) {
+          if (buffer.length > 0) {
+            // There is no data remaining in the file. Get ready to write the final separator on
+            // the next iteration.
+            buffer = finalSeparator;
+            pos = 0;
+          } else {
+            // We never read data into the buffer. This must be an empty file.
+            // Immediate EOF, no separator needed.
+            buffer = null;
+            return -1;
+          }
+        } else {
+          // Read the next data from the file into the buffer.
+          writer.write(fileReader.next(), encoder);
+          encoder.flush();
+          swapBuffer();
+        }
+      }
+
+      return bytesRead;
+    }
+
+    private void swapBuffer() {
       pos = 0;
       buffer = output.toByteArray();
       output.reset();
-      return read();
     }
 
     /**
@@ -323,4 +426,14 @@ class Display extends FsCommand {
       super.close();
     }
   }
+
+  private static void validateInputStreamReadArguments(byte[] dest, int destPos, int destLen)
+      throws IOException {
+    if (dest == null) {
+      throw new NullPointerException("null destination buffer");
+    } else if (destPos < 0 || destLen < 0 || destLen > dest.length - destPos) {
+      throw new IndexOutOfBoundsException(String.format(
+          "invalid destination buffer range: destPos = %d, destLen = %d", destPos, destLen));
+    }
+  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java

@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.shell;
 
-import static org.junit.Assert.*;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -34,7 +34,10 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.assertj.core.api.Assertions;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 
 /**
  * This class tests the logic for displaying the binary formats supported
@@ -44,62 +47,253 @@ public class TestTextCommand {
   private static final File TEST_ROOT_DIR =
       GenericTestUtils.getTestDir("testText");
   private static final String AVRO_FILENAME =
-    new File(TEST_ROOT_DIR, "weather.avro").toURI().getPath();
+      new File(TEST_ROOT_DIR, "weather.avro").toURI().getPath();
   private static final String TEXT_FILENAME =
-    new File(TEST_ROOT_DIR, "testtextfile.txt").toURI().getPath();
+      new File(TEST_ROOT_DIR, "testtextfile.txt").toURI().getPath();
+  private static final String SEQUENCE_FILENAME =
+      new File(TEST_ROOT_DIR, "NonWritableSequenceFile").toURI().getPath();
 
   private static final String SEPARATOR = System.getProperty("line.separator");
 
+  private static final String AVRO_EXPECTED_OUTPUT =
+      "{\"station\":\"011990-99999\",\"time\":-619524000000,\"temp\":0}" + SEPARATOR
+      + "{\"station\":\"011990-99999\",\"time\":-619506000000,\"temp\":22}" + SEPARATOR
+      + "{\"station\":\"011990-99999\",\"time\":-619484400000,\"temp\":-11}" + SEPARATOR
+      + "{\"station\":\"012650-99999\",\"time\":-655531200000,\"temp\":111}" + SEPARATOR
+      + "{\"station\":\"012650-99999\",\"time\":-655509600000,\"temp\":78}" + SEPARATOR;
+
+  private static final String SEQUENCE_FILE_EXPECTED_OUTPUT =
+      "Key1\tValue1" + SEPARATOR + "Key2\tValue2" + SEPARATOR;
+
+  @Rule
+  public final Timeout testTimeout = new Timeout(30000);
+
   /**
    * Tests whether binary Avro data files are displayed correctly.
    */
-  @Test (timeout = 30000)
+  @Test
   public void testDisplayForAvroFiles() throws Exception {
-    String expectedOutput =
-        "{\"station\":\"011990-99999\",\"time\":-619524000000,\"temp\":0}" + SEPARATOR
-        + "{\"station\":\"011990-99999\",\"time\":-619506000000,\"temp\":22}" + SEPARATOR
-        + "{\"station\":\"011990-99999\",\"time\":-619484400000,\"temp\":-11}" + SEPARATOR
-        + "{\"station\":\"012650-99999\",\"time\":-655531200000,\"temp\":111}" + SEPARATOR
-        + "{\"station\":\"012650-99999\",\"time\":-655509600000,\"temp\":78}" + SEPARATOR;
-
     String output = readUsingTextCommand(AVRO_FILENAME,
                                          generateWeatherAvroBinaryData());
-    assertEquals(expectedOutput, output);
+    Assertions.assertThat(output).describedAs("output").isEqualTo(AVRO_EXPECTED_OUTPUT);
+  }
+
+  @Test
+  public void testDisplayForAvroFilesSmallMultiByteReads() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setInt(IO_FILE_BUFFER_SIZE_KEY, 2);
+    createFile(AVRO_FILENAME, generateWeatherAvroBinaryData());
+    URI uri = new URI(AVRO_FILENAME);
+    String output = readUsingTextCommand(uri, conf);
+    Assertions.assertThat(output).describedAs("output").isEqualTo(AVRO_EXPECTED_OUTPUT);
+  }
+
+  @Test
+  public void testEmptyAvroFile() throws Exception {
+    String output = readUsingTextCommand(AVRO_FILENAME,
+                                         generateEmptyAvroBinaryData());
+    Assertions.assertThat(output).describedAs("output").isEmpty();
+  }
+
+  @Test(expected = NullPointerException.class)
+  public void testAvroFileInputStreamNullBuffer() throws Exception {
+    createFile(AVRO_FILENAME, generateWeatherAvroBinaryData());
+    URI uri = new URI(AVRO_FILENAME);
+    Configuration conf = new Configuration();
+    try (InputStream is = getInputStream(uri, conf)) {
+      is.read(null, 0, 10);
+    }
+  }
+
+  @Test(expected = IndexOutOfBoundsException.class)
+  public void testAvroFileInputStreamNegativePosition() throws Exception {
+    createFile(AVRO_FILENAME, generateWeatherAvroBinaryData());
+    URI uri = new URI(AVRO_FILENAME);
+    Configuration conf = new Configuration();
+    try (InputStream is = getInputStream(uri, conf)) {
+      is.read(new byte[10], -1, 10);
+    }
+  }
+
+  @Test(expected = IndexOutOfBoundsException.class)
+  public void testAvroFileInputStreamTooLong() throws Exception {
+    createFile(AVRO_FILENAME, generateWeatherAvroBinaryData());
+    URI uri = new URI(AVRO_FILENAME);
+    Configuration conf = new Configuration();
+    try (InputStream is = getInputStream(uri, conf)) {
+      is.read(new byte[10], 0, 11);
+    }
+  }
+
+  @Test
+  public void testAvroFileInputStreamZeroLengthRead() throws Exception {
+    createFile(AVRO_FILENAME, generateWeatherAvroBinaryData());
+    URI uri = new URI(AVRO_FILENAME);
+    Configuration conf = new Configuration();
+    try (InputStream is = getInputStream(uri, conf)) {
+      Assertions.assertThat(is.read(new byte[10], 0, 0)).describedAs("bytes read").isEqualTo(0);
+    }
+  }
+
+  @Test
+  public void testAvroFileInputStreamConsistentEOF() throws Exception {
+    createFile(AVRO_FILENAME, generateWeatherAvroBinaryData());
+    URI uri = new URI(AVRO_FILENAME);
+    Configuration conf = new Configuration();
+    try (InputStream is = getInputStream(uri, conf)) {
+      inputStreamToString(is);
+      Assertions.assertThat(is.read()).describedAs("single byte EOF").isEqualTo(-1);
+      Assertions.assertThat(is.read(new byte[10], 0, 10)).describedAs("multi byte EOF")
+              .isEqualTo(-1);
+    }
+  }
+
+  @Test
+  public void testAvroFileInputStreamSingleAndMultiByteReads() throws Exception {
+    createFile(AVRO_FILENAME, generateWeatherAvroBinaryData());
+    URI uri = new URI(AVRO_FILENAME);
+    Configuration conf = new Configuration();
+    try (InputStream is1 = getInputStream(uri, conf);
+        InputStream is2 = getInputStream(uri, conf)) {
+      String multiByteReads = inputStreamToString(is1);
+      String singleByteReads = inputStreamSingleByteReadsToString(is2);
+      Assertions.assertThat(multiByteReads)
+          .describedAs("same bytes read from multi and single byte reads")
+          .isEqualTo(singleByteReads);
+    }
   }
 
   /**
    * Tests that a zero-length file is displayed correctly.
    */
-  @Test (timeout = 30000)
-  public void testEmptyTextFil() throws Exception {
-    byte[] emptyContents = { };
+  @Test
+  public void testEmptyTextFile() throws Exception {
+    byte[] emptyContents = {};
     String output = readUsingTextCommand(TEXT_FILENAME, emptyContents);
-    assertTrue("".equals(output));
+    Assertions.assertThat(output).describedAs("output").isEmpty();
   }
 
   /**
    * Tests that a one-byte file is displayed correctly.
    */
-  @Test (timeout = 30000)
-  public void testOneByteTextFil() throws Exception {
-    byte[] oneByteContents = { 'x' };
+  @Test
+  public void testOneByteTextFile() throws Exception {
+    byte[] oneByteContents = {'x'};
     String output = readUsingTextCommand(TEXT_FILENAME, oneByteContents);
-    assertTrue(new String(oneByteContents).equals(output));
+    String expected = new String(oneByteContents, StandardCharsets.UTF_8);
+    Assertions.assertThat(output).describedAs("output").isEqualTo(expected);
   }
 
   /**
-   * Tests that a one-byte file is displayed correctly.
+   * Tests that a two-byte file is displayed correctly.
    */
-  @Test (timeout = 30000)
-  public void testTwoByteTextFil() throws Exception {
-    byte[] twoByteContents = { 'x', 'y' };
+  @Test
+  public void testTwoByteTextFile() throws Exception {
+    byte[] twoByteContents = {'x', 'y'};
     String output = readUsingTextCommand(TEXT_FILENAME, twoByteContents);
-    assertTrue(new String(twoByteContents).equals(output));
+    String expected = new String(twoByteContents, StandardCharsets.UTF_8);
+    Assertions.assertThat(output).describedAs("output").isEqualTo(expected);
+  }
+
+  @Test
+  public void testDisplayForNonWritableSequenceFile() throws Exception {
+    Configuration conf = new Configuration();
+    createNonWritableSequenceFile(SEQUENCE_FILENAME, conf);
+    URI uri = new URI(SEQUENCE_FILENAME);
+    String output = readUsingTextCommand(uri, conf);
+    Assertions.assertThat(output).describedAs("output").isEqualTo(SEQUENCE_FILE_EXPECTED_OUTPUT);
+  }
+
+  @Test
+  public void testDisplayForSequenceFileSmallMultiByteReads() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setInt(IO_FILE_BUFFER_SIZE_KEY, 2);
+    createNonWritableSequenceFile(SEQUENCE_FILENAME, conf);
+    URI uri = new URI(SEQUENCE_FILENAME);
+    String output = readUsingTextCommand(uri, conf);
+    Assertions.assertThat(output).describedAs("output").isEqualTo(SEQUENCE_FILE_EXPECTED_OUTPUT);
+  }
+
+  @Test
+  public void testEmptySequenceFile() throws Exception {
+    Configuration conf = new Configuration();
+    createEmptySequenceFile(SEQUENCE_FILENAME, conf);
+    URI uri = new URI(SEQUENCE_FILENAME);
+    String output = readUsingTextCommand(uri, conf);
+    Assertions.assertThat(output).describedAs("output").isEmpty();
+  }
+
+  @Test(expected = NullPointerException.class)
+  public void testSequenceFileInputStreamNullBuffer() throws Exception {
+    Configuration conf = new Configuration();
+    createNonWritableSequenceFile(SEQUENCE_FILENAME, conf);
+    URI uri = new URI(SEQUENCE_FILENAME);
+    try (InputStream is = getInputStream(uri, conf)) {
+      is.read(null, 0, 10);
+    }
+  }
+
+  @Test(expected = IndexOutOfBoundsException.class)
+  public void testSequenceFileInputStreamNegativePosition() throws Exception {
+    Configuration conf = new Configuration();
+    createNonWritableSequenceFile(SEQUENCE_FILENAME, conf);
+    URI uri = new URI(SEQUENCE_FILENAME);
+    try (InputStream is = getInputStream(uri, conf)) {
+      is.read(new byte[10], -1, 10);
+    }
+  }
+
+  @Test(expected = IndexOutOfBoundsException.class)
+  public void testSequenceFileInputStreamTooLong() throws Exception {
+    Configuration conf = new Configuration();
+    createNonWritableSequenceFile(SEQUENCE_FILENAME, conf);
+    URI uri = new URI(SEQUENCE_FILENAME);
+    try (InputStream is = getInputStream(uri, conf)) {
+      is.read(new byte[10], 0, 11);
+    }
+  }
+
+  @Test
+  public void testSequenceFileInputStreamZeroLengthRead() throws Exception {
+    Configuration conf = new Configuration();
+    createNonWritableSequenceFile(SEQUENCE_FILENAME, conf);
+    URI uri = new URI(SEQUENCE_FILENAME);
+    try (InputStream is = getInputStream(uri, conf)) {
+      Assertions.assertThat(is.read(new byte[10], 0, 0)).describedAs("bytes read").isEqualTo(0);
+    }
+  }
+
+  @Test
+  public void testSequenceFileInputStreamConsistentEOF() throws Exception {
+    Configuration conf = new Configuration();
+    createNonWritableSequenceFile(SEQUENCE_FILENAME, conf);
+    URI uri = new URI(SEQUENCE_FILENAME);
+    try (InputStream is = getInputStream(uri, conf)) {
+      inputStreamToString(is);
+      Assertions.assertThat(is.read()).describedAs("single byte EOF").isEqualTo(-1);
+      Assertions.assertThat(is.read(new byte[10], 0, 10)).describedAs("multi byte EOF")
+              .isEqualTo(-1);
+    }
+  }
+
+  @Test
+  public void testSequenceFileInputStreamSingleAndMultiByteReads() throws Exception {
+    Configuration conf = new Configuration();
+    createNonWritableSequenceFile(SEQUENCE_FILENAME, conf);
+    URI uri = new URI(SEQUENCE_FILENAME);
+    try (InputStream is1 = getInputStream(uri, conf);
+        InputStream is2 = getInputStream(uri, conf)) {
+      String multiByteReads = inputStreamToString(is1);
+      String singleByteReads = inputStreamSingleByteReadsToString(is2);
+      Assertions.assertThat(multiByteReads)
+          .describedAs("same bytes read from multi and single byte reads")
+          .isEqualTo(singleByteReads);
+    }
   }
 
   // Create a file on the local file system and read it using
   // the Display.Text class.
-  private String readUsingTextCommand(String fileName, byte[] fileContents)
+  private static String readUsingTextCommand(String fileName, byte[] fileContents)
           throws Exception {
     createFile(fileName, fileContents);
 
@@ -108,29 +302,27 @@ public class TestTextCommand {
     return readUsingTextCommand(localPath, conf);
   }
   // Read a file using Display.Text class.
-  private String readUsingTextCommand(URI uri, Configuration conf)
+  private static String readUsingTextCommand(URI uri, Configuration conf)
       throws Exception {
-    // Prepare and call the Text command's protected getInputStream method
-    // using reflection.
-    PathData pathData = new PathData(uri, conf);
-    Display.Text text = new Display.Text() {
-      @Override
-      public InputStream getInputStream(PathData item) throws IOException {
-        return super.getInputStream(item);
-      }
-    };
-    text.setConf(conf);
-    InputStream stream = text.getInputStream(pathData);
+    InputStream stream = getInputStream(uri, conf);
     return inputStreamToString(stream);
   }
 
-  private String inputStreamToString(InputStream stream) throws IOException {
+  private static String inputStreamToString(InputStream stream) throws IOException {
     StringWriter writer = new StringWriter();
     IOUtils.copy(stream, writer, StandardCharsets.UTF_8);
     return writer.toString();
   }
 
-  private void createFile(String fileName, byte[] contents) throws IOException {
+  private static String inputStreamSingleByteReadsToString(InputStream stream) throws IOException {
+    StringWriter writer = new StringWriter();
+    for (int b = stream.read(); b != -1; b = stream.read()) {
+      writer.write(b);
+    }
+    return writer.toString();
+  }
+
+  private static void createFile(String fileName, byte[] contents) throws IOException {
     Files.createDirectories(TEST_ROOT_DIR.toPath());
     File file = new File(fileName);
     file.createNewFile();
@@ -139,118 +331,207 @@ public class TestTextCommand {
     stream.close();
   }
 
-  private byte[] generateWeatherAvroBinaryData() {
+  private static byte[] generateWeatherAvroBinaryData() {
     // The contents of a simple binary Avro file with weather records.
     byte[] contents = {
-      (byte) 0x4f, (byte) 0x62, (byte) 0x6a, (byte)  0x1,
-      (byte)  0x4, (byte) 0x14, (byte) 0x61, (byte) 0x76,
-      (byte) 0x72, (byte) 0x6f, (byte) 0x2e, (byte) 0x63,
-      (byte) 0x6f, (byte) 0x64, (byte) 0x65, (byte) 0x63,
-      (byte)  0x8, (byte) 0x6e, (byte) 0x75, (byte) 0x6c,
-      (byte) 0x6c, (byte) 0x16, (byte) 0x61, (byte) 0x76,
-      (byte) 0x72, (byte) 0x6f, (byte) 0x2e, (byte) 0x73,
-      (byte) 0x63, (byte) 0x68, (byte) 0x65, (byte) 0x6d,
-      (byte) 0x61, (byte) 0xf2, (byte)  0x2, (byte) 0x7b,
-      (byte) 0x22, (byte) 0x74, (byte) 0x79, (byte) 0x70,
-      (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
-      (byte) 0x72, (byte) 0x65, (byte) 0x63, (byte) 0x6f,
-      (byte) 0x72, (byte) 0x64, (byte) 0x22, (byte) 0x2c,
-      (byte) 0x22, (byte) 0x6e, (byte) 0x61, (byte) 0x6d,
-      (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
-      (byte) 0x57, (byte) 0x65, (byte) 0x61, (byte) 0x74,
-      (byte) 0x68, (byte) 0x65, (byte) 0x72, (byte) 0x22,
-      (byte) 0x2c, (byte) 0x22, (byte) 0x6e, (byte) 0x61,
-      (byte) 0x6d, (byte) 0x65, (byte) 0x73, (byte) 0x70,
-      (byte) 0x61, (byte) 0x63, (byte) 0x65, (byte) 0x22,
-      (byte) 0x3a, (byte) 0x22, (byte) 0x74, (byte) 0x65,
-      (byte) 0x73, (byte) 0x74, (byte) 0x22, (byte) 0x2c,
-      (byte) 0x22, (byte) 0x66, (byte) 0x69, (byte) 0x65,
-      (byte) 0x6c, (byte) 0x64, (byte) 0x73, (byte) 0x22,
-      (byte) 0x3a, (byte) 0x5b, (byte) 0x7b, (byte) 0x22,
-      (byte) 0x6e, (byte) 0x61, (byte) 0x6d, (byte) 0x65,
-      (byte) 0x22, (byte) 0x3a, (byte) 0x22, (byte) 0x73,
-      (byte) 0x74, (byte) 0x61, (byte) 0x74, (byte) 0x69,
-      (byte) 0x6f, (byte) 0x6e, (byte) 0x22, (byte) 0x2c,
-      (byte) 0x22, (byte) 0x74, (byte) 0x79, (byte) 0x70,
-      (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
-      (byte) 0x73, (byte) 0x74, (byte) 0x72, (byte) 0x69,
-      (byte) 0x6e, (byte) 0x67, (byte) 0x22, (byte) 0x7d,
-      (byte) 0x2c, (byte) 0x7b, (byte) 0x22, (byte) 0x6e,
-      (byte) 0x61, (byte) 0x6d, (byte) 0x65, (byte) 0x22,
-      (byte) 0x3a, (byte) 0x22, (byte) 0x74, (byte) 0x69,
-      (byte) 0x6d, (byte) 0x65, (byte) 0x22, (byte) 0x2c,
-      (byte) 0x22, (byte) 0x74, (byte) 0x79, (byte) 0x70,
-      (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
-      (byte) 0x6c, (byte) 0x6f, (byte) 0x6e, (byte) 0x67,
-      (byte) 0x22, (byte) 0x7d, (byte) 0x2c, (byte) 0x7b,
-      (byte) 0x22, (byte) 0x6e, (byte) 0x61, (byte) 0x6d,
-      (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
-      (byte) 0x74, (byte) 0x65, (byte) 0x6d, (byte) 0x70,
-      (byte) 0x22, (byte) 0x2c, (byte) 0x22, (byte) 0x74,
-      (byte) 0x79, (byte) 0x70, (byte) 0x65, (byte) 0x22,
-      (byte) 0x3a, (byte) 0x22, (byte) 0x69, (byte) 0x6e,
-      (byte) 0x74, (byte) 0x22, (byte) 0x7d, (byte) 0x5d,
-      (byte) 0x2c, (byte) 0x22, (byte) 0x64, (byte) 0x6f,
-      (byte) 0x63, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
-      (byte) 0x41, (byte) 0x20, (byte) 0x77, (byte) 0x65,
-      (byte) 0x61, (byte) 0x74, (byte) 0x68, (byte) 0x65,
-      (byte) 0x72, (byte) 0x20, (byte) 0x72, (byte) 0x65,
-      (byte) 0x61, (byte) 0x64, (byte) 0x69, (byte) 0x6e,
-      (byte) 0x67, (byte) 0x2e, (byte) 0x22, (byte) 0x7d,
-      (byte)  0x0, (byte) 0xb0, (byte) 0x81, (byte) 0xb3,
-      (byte) 0xc4, (byte)  0xa, (byte)  0xc, (byte) 0xf6,
-      (byte) 0x62, (byte) 0xfa, (byte) 0xc9, (byte) 0x38,
-      (byte) 0xfd, (byte) 0x7e, (byte) 0x52, (byte)  0x0,
-      (byte) 0xa7, (byte)  0xa, (byte) 0xcc, (byte)  0x1,
-      (byte) 0x18, (byte) 0x30, (byte) 0x31, (byte) 0x31,
-      (byte) 0x39, (byte) 0x39, (byte) 0x30, (byte) 0x2d,
-      (byte) 0x39, (byte) 0x39, (byte) 0x39, (byte) 0x39,
-      (byte) 0x39, (byte) 0xff, (byte) 0xa3, (byte) 0x90,
-      (byte) 0xe8, (byte) 0x87, (byte) 0x24, (byte)  0x0,
-      (byte) 0x18, (byte) 0x30, (byte) 0x31, (byte) 0x31,
-      (byte) 0x39, (byte) 0x39, (byte) 0x30, (byte) 0x2d,
-      (byte) 0x39, (byte) 0x39, (byte) 0x39, (byte) 0x39,
-      (byte) 0x39, (byte) 0xff, (byte) 0x81, (byte) 0xfb,
-      (byte) 0xd6, (byte) 0x87, (byte) 0x24, (byte) 0x2c,
-      (byte) 0x18, (byte) 0x30, (byte) 0x31, (byte) 0x31,
-      (byte) 0x39, (byte) 0x39, (byte) 0x30, (byte) 0x2d,
-      (byte) 0x39, (byte) 0x39, (byte) 0x39, (byte) 0x39,
-      (byte) 0x39, (byte) 0xff, (byte) 0xa5, (byte) 0xae,
-      (byte) 0xc2, (byte) 0x87, (byte) 0x24, (byte) 0x15,
-      (byte) 0x18, (byte) 0x30, (byte) 0x31, (byte) 0x32,
-      (byte) 0x36, (byte) 0x35, (byte) 0x30, (byte) 0x2d,
-      (byte) 0x39, (byte) 0x39, (byte) 0x39, (byte) 0x39,
-      (byte) 0x39, (byte) 0xff, (byte) 0xb7, (byte) 0xa2,
-      (byte) 0x8b, (byte) 0x94, (byte) 0x26, (byte) 0xde,
-      (byte)  0x1, (byte) 0x18, (byte) 0x30, (byte) 0x31,
-      (byte) 0x32, (byte) 0x36, (byte) 0x35, (byte) 0x30,
-      (byte) 0x2d, (byte) 0x39, (byte) 0x39, (byte) 0x39,
-      (byte) 0x39, (byte) 0x39, (byte) 0xff, (byte) 0xdb,
-      (byte) 0xd5, (byte) 0xf6, (byte) 0x93, (byte) 0x26,
-      (byte) 0x9c, (byte)  0x1, (byte) 0xb0, (byte) 0x81,
-      (byte) 0xb3, (byte) 0xc4, (byte)  0xa, (byte)  0xc,
-      (byte) 0xf6, (byte) 0x62, (byte) 0xfa, (byte) 0xc9,
-      (byte) 0x38, (byte) 0xfd, (byte) 0x7e, (byte) 0x52,
-      (byte)  0x0, (byte) 0xa7,
+        (byte) 0x4f, (byte) 0x62, (byte) 0x6a, (byte)  0x1,
+        (byte)  0x4, (byte) 0x14, (byte) 0x61, (byte) 0x76,
+        (byte) 0x72, (byte) 0x6f, (byte) 0x2e, (byte) 0x63,
+        (byte) 0x6f, (byte) 0x64, (byte) 0x65, (byte) 0x63,
+        (byte)  0x8, (byte) 0x6e, (byte) 0x75, (byte) 0x6c,
+        (byte) 0x6c, (byte) 0x16, (byte) 0x61, (byte) 0x76,
+        (byte) 0x72, (byte) 0x6f, (byte) 0x2e, (byte) 0x73,
+        (byte) 0x63, (byte) 0x68, (byte) 0x65, (byte) 0x6d,
+        (byte) 0x61, (byte) 0xf2, (byte)  0x2, (byte) 0x7b,
+        (byte) 0x22, (byte) 0x74, (byte) 0x79, (byte) 0x70,
+        (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
+        (byte) 0x72, (byte) 0x65, (byte) 0x63, (byte) 0x6f,
+        (byte) 0x72, (byte) 0x64, (byte) 0x22, (byte) 0x2c,
+        (byte) 0x22, (byte) 0x6e, (byte) 0x61, (byte) 0x6d,
+        (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
+        (byte) 0x57, (byte) 0x65, (byte) 0x61, (byte) 0x74,
+        (byte) 0x68, (byte) 0x65, (byte) 0x72, (byte) 0x22,
+        (byte) 0x2c, (byte) 0x22, (byte) 0x6e, (byte) 0x61,
+        (byte) 0x6d, (byte) 0x65, (byte) 0x73, (byte) 0x70,
+        (byte) 0x61, (byte) 0x63, (byte) 0x65, (byte) 0x22,
+        (byte) 0x3a, (byte) 0x22, (byte) 0x74, (byte) 0x65,
+        (byte) 0x73, (byte) 0x74, (byte) 0x22, (byte) 0x2c,
+        (byte) 0x22, (byte) 0x66, (byte) 0x69, (byte) 0x65,
+        (byte) 0x6c, (byte) 0x64, (byte) 0x73, (byte) 0x22,
+        (byte) 0x3a, (byte) 0x5b, (byte) 0x7b, (byte) 0x22,
+        (byte) 0x6e, (byte) 0x61, (byte) 0x6d, (byte) 0x65,
+        (byte) 0x22, (byte) 0x3a, (byte) 0x22, (byte) 0x73,
+        (byte) 0x74, (byte) 0x61, (byte) 0x74, (byte) 0x69,
+        (byte) 0x6f, (byte) 0x6e, (byte) 0x22, (byte) 0x2c,
+        (byte) 0x22, (byte) 0x74, (byte) 0x79, (byte) 0x70,
+        (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
+        (byte) 0x73, (byte) 0x74, (byte) 0x72, (byte) 0x69,
+        (byte) 0x6e, (byte) 0x67, (byte) 0x22, (byte) 0x7d,
+        (byte) 0x2c, (byte) 0x7b, (byte) 0x22, (byte) 0x6e,
+        (byte) 0x61, (byte) 0x6d, (byte) 0x65, (byte) 0x22,
+        (byte) 0x3a, (byte) 0x22, (byte) 0x74, (byte) 0x69,
+        (byte) 0x6d, (byte) 0x65, (byte) 0x22, (byte) 0x2c,
+        (byte) 0x22, (byte) 0x74, (byte) 0x79, (byte) 0x70,
+        (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
+        (byte) 0x6c, (byte) 0x6f, (byte) 0x6e, (byte) 0x67,
+        (byte) 0x22, (byte) 0x7d, (byte) 0x2c, (byte) 0x7b,
+        (byte) 0x22, (byte) 0x6e, (byte) 0x61, (byte) 0x6d,
+        (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
+        (byte) 0x74, (byte) 0x65, (byte) 0x6d, (byte) 0x70,
+        (byte) 0x22, (byte) 0x2c, (byte) 0x22, (byte) 0x74,
+        (byte) 0x79, (byte) 0x70, (byte) 0x65, (byte) 0x22,
+        (byte) 0x3a, (byte) 0x22, (byte) 0x69, (byte) 0x6e,
+        (byte) 0x74, (byte) 0x22, (byte) 0x7d, (byte) 0x5d,
+        (byte) 0x2c, (byte) 0x22, (byte) 0x64, (byte) 0x6f,
+        (byte) 0x63, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
+        (byte) 0x41, (byte) 0x20, (byte) 0x77, (byte) 0x65,
+        (byte) 0x61, (byte) 0x74, (byte) 0x68, (byte) 0x65,
+        (byte) 0x72, (byte) 0x20, (byte) 0x72, (byte) 0x65,
+        (byte) 0x61, (byte) 0x64, (byte) 0x69, (byte) 0x6e,
+        (byte) 0x67, (byte) 0x2e, (byte) 0x22, (byte) 0x7d,
+        (byte)  0x0, (byte) 0xb0, (byte) 0x81, (byte) 0xb3,
+        (byte) 0xc4, (byte)  0xa, (byte)  0xc, (byte) 0xf6,
+        (byte) 0x62, (byte) 0xfa, (byte) 0xc9, (byte) 0x38,
+        (byte) 0xfd, (byte) 0x7e, (byte) 0x52, (byte)  0x0,
+        (byte) 0xa7, (byte)  0xa, (byte) 0xcc, (byte)  0x1,
+        (byte) 0x18, (byte) 0x30, (byte) 0x31, (byte) 0x31,
+        (byte) 0x39, (byte) 0x39, (byte) 0x30, (byte) 0x2d,
+        (byte) 0x39, (byte) 0x39, (byte) 0x39, (byte) 0x39,
+        (byte) 0x39, (byte) 0xff, (byte) 0xa3, (byte) 0x90,
+        (byte) 0xe8, (byte) 0x87, (byte) 0x24, (byte)  0x0,
+        (byte) 0x18, (byte) 0x30, (byte) 0x31, (byte) 0x31,
+        (byte) 0x39, (byte) 0x39, (byte) 0x30, (byte) 0x2d,
+        (byte) 0x39, (byte) 0x39, (byte) 0x39, (byte) 0x39,
+        (byte) 0x39, (byte) 0xff, (byte) 0x81, (byte) 0xfb,
+        (byte) 0xd6, (byte) 0x87, (byte) 0x24, (byte) 0x2c,
+        (byte) 0x18, (byte) 0x30, (byte) 0x31, (byte) 0x31,
+        (byte) 0x39, (byte) 0x39, (byte) 0x30, (byte) 0x2d,
+        (byte) 0x39, (byte) 0x39, (byte) 0x39, (byte) 0x39,
+        (byte) 0x39, (byte) 0xff, (byte) 0xa5, (byte) 0xae,
+        (byte) 0xc2, (byte) 0x87, (byte) 0x24, (byte) 0x15,
+        (byte) 0x18, (byte) 0x30, (byte) 0x31, (byte) 0x32,
+        (byte) 0x36, (byte) 0x35, (byte) 0x30, (byte) 0x2d,
+        (byte) 0x39, (byte) 0x39, (byte) 0x39, (byte) 0x39,
+        (byte) 0x39, (byte) 0xff, (byte) 0xb7, (byte) 0xa2,
+        (byte) 0x8b, (byte) 0x94, (byte) 0x26, (byte) 0xde,
+        (byte)  0x1, (byte) 0x18, (byte) 0x30, (byte) 0x31,
+        (byte) 0x32, (byte) 0x36, (byte) 0x35, (byte) 0x30,
+        (byte) 0x2d, (byte) 0x39, (byte) 0x39, (byte) 0x39,
+        (byte) 0x39, (byte) 0x39, (byte) 0xff, (byte) 0xdb,
+        (byte) 0xd5, (byte) 0xf6, (byte) 0x93, (byte) 0x26,
+        (byte) 0x9c, (byte)  0x1, (byte) 0xb0, (byte) 0x81,
+        (byte) 0xb3, (byte) 0xc4, (byte)  0xa, (byte)  0xc,
+        (byte) 0xf6, (byte) 0x62, (byte) 0xfa, (byte) 0xc9,
+        (byte) 0x38, (byte) 0xfd, (byte) 0x7e, (byte) 0x52,
+        (byte)  0x0, (byte) 0xa7,
     };
 
     return contents;
   }
 
-  @Test
-  public void testDisplayForNonWritableSequenceFile() throws Exception {
-    Configuration conf = new Configuration();
+  private static byte[] generateEmptyAvroBinaryData() {
+    // The binary contents of an empty Avro file (no records).
+    byte[] contents = new byte[] {
+        (byte) 0x4f, (byte) 0x62, (byte) 0x6a, (byte) 0x01,
+        (byte) 0x04, (byte) 0x16, (byte) 0x61, (byte) 0x76,
+        (byte) 0x72, (byte) 0x6f, (byte) 0x2e, (byte) 0x73,
+        (byte) 0x63, (byte) 0x68, (byte) 0x65, (byte) 0x6d,
+        (byte) 0x61, (byte) 0x92, (byte) 0x03, (byte) 0x7b,
+        (byte) 0x22, (byte) 0x74, (byte) 0x79, (byte) 0x70,
+        (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
+        (byte) 0x72, (byte) 0x65, (byte) 0x63, (byte) 0x6f,
+        (byte) 0x72, (byte) 0x64, (byte) 0x22, (byte) 0x2c,
+        (byte) 0x22, (byte) 0x6e, (byte) 0x61, (byte) 0x6d,
+        (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
+        (byte) 0x55, (byte) 0x73, (byte) 0x65, (byte) 0x72,
+        (byte) 0x22, (byte) 0x2c, (byte) 0x22, (byte) 0x6e,
+        (byte) 0x61, (byte) 0x6d, (byte) 0x65, (byte) 0x73,
+        (byte) 0x70, (byte) 0x61, (byte) 0x63, (byte) 0x65,
+        (byte) 0x22, (byte) 0x3a, (byte) 0x22, (byte) 0x65,
+        (byte) 0x78, (byte) 0x61, (byte) 0x6d, (byte) 0x70,
+        (byte) 0x6c, (byte) 0x65, (byte) 0x2e, (byte) 0x61,
+        (byte) 0x76, (byte) 0x72, (byte) 0x6f, (byte) 0x22,
+        (byte) 0x2c, (byte) 0x22, (byte) 0x66, (byte) 0x69,
+        (byte) 0x65, (byte) 0x6c, (byte) 0x64, (byte) 0x73,
+        (byte) 0x22, (byte) 0x3a, (byte) 0x5b, (byte) 0x7b,
+        (byte) 0x22, (byte) 0x6e, (byte) 0x61, (byte) 0x6d,
+        (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
+        (byte) 0x6e, (byte) 0x61, (byte) 0x6d, (byte) 0x65,
+        (byte) 0x22, (byte) 0x2c, (byte) 0x22, (byte) 0x74,
+        (byte) 0x79, (byte) 0x70, (byte) 0x65, (byte) 0x22,
+        (byte) 0x3a, (byte) 0x22, (byte) 0x73, (byte) 0x74,
+        (byte) 0x72, (byte) 0x69, (byte) 0x6e, (byte) 0x67,
+        (byte) 0x22, (byte) 0x7d, (byte) 0x2c, (byte) 0x7b,
+        (byte) 0x22, (byte) 0x6e, (byte) 0x61, (byte) 0x6d,
+        (byte) 0x65, (byte) 0x22, (byte) 0x3a, (byte) 0x22,
+        (byte) 0x66, (byte) 0x61, (byte) 0x76, (byte) 0x6f,
+        (byte) 0x72, (byte) 0x69, (byte) 0x74, (byte) 0x65,
+        (byte) 0x5f, (byte) 0x6e, (byte) 0x75, (byte) 0x6d,
+        (byte) 0x62, (byte) 0x65, (byte) 0x72, (byte) 0x22,
+        (byte) 0x2c, (byte) 0x22, (byte) 0x74, (byte) 0x79,
+        (byte) 0x70, (byte) 0x65, (byte) 0x22, (byte) 0x3a,
+        (byte) 0x5b, (byte) 0x22, (byte) 0x69, (byte) 0x6e,
+        (byte) 0x74, (byte) 0x22, (byte) 0x2c, (byte) 0x22,
+        (byte) 0x6e, (byte) 0x75, (byte) 0x6c, (byte) 0x6c,
+        (byte) 0x22, (byte) 0x5d, (byte) 0x7d, (byte) 0x2c,
+        (byte) 0x7b, (byte) 0x22, (byte) 0x6e, (byte) 0x61,
+        (byte) 0x6d, (byte) 0x65, (byte) 0x22, (byte) 0x3a,
+        (byte) 0x22, (byte) 0x66, (byte) 0x61, (byte) 0x76,
+        (byte) 0x6f, (byte) 0x72, (byte) 0x69, (byte) 0x74,
+        (byte) 0x65, (byte) 0x5f, (byte) 0x63, (byte) 0x6f,
+        (byte) 0x6c, (byte) 0x6f, (byte) 0x72, (byte) 0x22,
+        (byte) 0x2c, (byte) 0x22, (byte) 0x74, (byte) 0x79,
+        (byte) 0x70, (byte) 0x65, (byte) 0x22, (byte) 0x3a,
+        (byte) 0x5b, (byte) 0x22, (byte) 0x73, (byte) 0x74,
+        (byte) 0x72, (byte) 0x69, (byte) 0x6e, (byte) 0x67,
+        (byte) 0x22, (byte) 0x2c, (byte) 0x22, (byte) 0x6e,
+        (byte) 0x75, (byte) 0x6c, (byte) 0x6c, (byte) 0x22,
+        (byte) 0x5d, (byte) 0x7d, (byte) 0x5d, (byte) 0x7d,
+        (byte) 0x14, (byte) 0x61, (byte) 0x76, (byte) 0x72,
+        (byte) 0x6f, (byte) 0x2e, (byte) 0x63, (byte) 0x6f,
+        (byte) 0x64, (byte) 0x65, (byte) 0x63, (byte) 0x0e,
+        (byte) 0x64, (byte) 0x65, (byte) 0x66, (byte) 0x6c,
+        (byte) 0x61, (byte) 0x74, (byte) 0x65, (byte) 0x00,
+        (byte) 0xed, (byte) 0xe0, (byte) 0xfa, (byte) 0x87,
+        (byte) 0x3c, (byte) 0x86, (byte) 0xf5, (byte) 0x5f,
+        (byte) 0x7d, (byte) 0x8d, (byte) 0x2f, (byte) 0xdb,
+        (byte) 0xc7, (byte) 0xe3, (byte) 0x11, (byte) 0x39,
+    };
+
+    return contents;
+  }
+
+  private static void createEmptySequenceFile(String fileName, Configuration conf)
+      throws IOException {
     conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization");
-    Path path = new Path(String.valueOf(TEST_ROOT_DIR), "NonWritableSequenceFile");
+    Path path = new Path(fileName);
     SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(path),
         SequenceFile.Writer.keyClass(String.class), SequenceFile.Writer.valueClass(String.class));
-    writer.append("Key1", "Value1");
-    writer.append("Key2", "Value2");
     writer.close();
-    String expected = "Key1\tValue1" + SEPARATOR + "Key2\tValue2" + SEPARATOR;
-    URI uri = path.toUri();
-    System.out.println(expected);
-    assertEquals(expected, readUsingTextCommand(uri, conf));
   }
-}
 
+  private static void createNonWritableSequenceFile(String fileName, Configuration conf)
+      throws IOException {
+    conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization");
+    Path path = new Path(fileName);
+    try (SequenceFile.Writer writer = SequenceFile.createWriter(conf,
+        SequenceFile.Writer.file(path), SequenceFile.Writer.keyClass(String.class),
+        SequenceFile.Writer.valueClass(String.class))) {
+      writer.append("Key1", "Value1");
+      writer.append("Key2", "Value2");
+    }
+  }
+
+  private static InputStream getInputStream(URI uri, Configuration conf) throws IOException {
+    // Prepare and call the Text command's protected getInputStream method.
+    PathData pathData = new PathData(uri, conf);
+    Display.Text text = new Display.Text() {
+      @Override
+      public InputStream getInputStream(PathData item) throws IOException {
+        return super.getInputStream(item);
+      }
+    };
+    text.setConf(conf);
+    return text.getInputStream(pathData);
+  }
+}