|
@@ -34,7 +34,7 @@ import java.util.HashSet;
|
|
|
import java.util.Set;
|
|
|
|
|
|
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
|
|
|
-import org.apache.commons.io.Charsets;
|
|
|
+import java.nio.charset.StandardCharsets;
|
|
|
import org.apache.hadoop.conf.Configuration;
|
|
|
import org.apache.hadoop.fs.FileSystem;
|
|
|
import org.apache.hadoop.fs.Path;
|
|
@@ -72,7 +72,7 @@ public class TestLineRecordReader {
|
|
|
String delimiter = conf.get("textinputformat.record.delimiter");
|
|
|
byte[] recordDelimiterBytes = null;
|
|
|
if (null != delimiter) {
|
|
|
- recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8);
|
|
|
+ recordDelimiterBytes = delimiter.getBytes(StandardCharsets.UTF_8);
|
|
|
}
|
|
|
// read the data without splitting to count the records
|
|
|
FileSplit split = new FileSplit(testFilePath, 0, testFileSize,
|
|
@@ -120,7 +120,7 @@ public class TestLineRecordReader {
|
|
|
String delimiter = conf.get("textinputformat.record.delimiter");
|
|
|
byte[] recordDelimiterBytes = null;
|
|
|
if (null != delimiter) {
|
|
|
- recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8);
|
|
|
+ recordDelimiterBytes = delimiter.getBytes(StandardCharsets.UTF_8);
|
|
|
}
|
|
|
// read the data without splitting to count the records
|
|
|
FileSplit split = new FileSplit(testFilePath, 0, testFileSize,
|
|
@@ -482,7 +482,7 @@ public class TestLineRecordReader {
|
|
|
String inputData = "abcdefghij++kl++mno";
|
|
|
Path inputFile = createInputFile(conf, inputData);
|
|
|
String delimiter = "++";
|
|
|
- byte[] recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8);
|
|
|
+ byte[] recordDelimiterBytes = delimiter.getBytes(StandardCharsets.UTF_8);
|
|
|
// the first split must contain two records to make sure that it also pulls
|
|
|
// in the record from the 2nd split
|
|
|
int splitLength = 15;
|
|
@@ -555,7 +555,7 @@ public class TestLineRecordReader {
|
|
|
inputData = "abcd|efgh|+|ij|kl|+|mno|pqr";
|
|
|
inputFile = createInputFile(conf, inputData);
|
|
|
delimiter = "|+|";
|
|
|
- recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8);
|
|
|
+ recordDelimiterBytes = delimiter.getBytes(StandardCharsets.UTF_8);
|
|
|
// walking over the buffer and split sizes checks for proper processing
|
|
|
// of the ambiguous bytes of the delimiter
|
|
|
for (int bufferSize = 1; bufferSize <= inputData.length(); bufferSize++) {
|