Browse Source

svn merge -c 1463221 FIXES: MAPREDUCE-4974. Optimising the LineRecordReader initialize() method (Gelesh via bobby)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1463224 13f79535-47bb-0310-9956-ffa450edef68
Robert Joseph Evans 12 years ago
parent
commit
be7e6f1ff2

+ 3 - 0
hadoop-mapreduce-project/CHANGES.txt

@@ -45,6 +45,9 @@ Release 0.23.7 - UNRELEASED
     MAPREDUCE-4822. Unnecessary conversions in History Events. (Chu Tong via
     MAPREDUCE-4822. Unnecessary conversions in History Events. (Chu Tong via
     jlowe)
     jlowe)
 
 
+    MAPREDUCE-4974. Optimising the LineRecordReader initialize() method
+    (Gelesh via bobby)
+
   BUG FIXES
   BUG FIXES
 
 
     MAPREDUCE-4278. cannot run two local jobs in parallel from the same 
     MAPREDUCE-4278. cannot run two local jobs in parallel from the same 

+ 3 - 6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java

@@ -81,13 +81,13 @@ public class LineRecordReader extends RecordReader<LongWritable, Text> {
     start = split.getStart();
     start = split.getStart();
     end = start + split.getLength();
     end = start + split.getLength();
     final Path file = split.getPath();
     final Path file = split.getPath();
-    compressionCodecs = new CompressionCodecFactory(job);
-    codec = compressionCodecs.getCodec(file);
 
 
     // open the file and seek to the start of the split
     // open the file and seek to the start of the split
     final FileSystem fs = file.getFileSystem(job);
     final FileSystem fs = file.getFileSystem(job);
     fileIn = fs.open(file);
     fileIn = fs.open(file);
     if (isCompressedInput()) {
     if (isCompressedInput()) {
+      compressionCodecs = new CompressionCodecFactory(job);
+      codec = compressionCodecs.getCodec(file);
       decompressor = CodecPool.getDecompressor(codec);
       decompressor = CodecPool.getDecompressor(codec);
       if (codec instanceof SplittableCompressionCodec) {
       if (codec instanceof SplittableCompressionCodec) {
         final SplitCompressionInputStream cIn =
         final SplitCompressionInputStream cIn =
@@ -166,9 +166,6 @@ public class LineRecordReader extends RecordReader<LongWritable, Text> {
     while (getFilePosition() <= end) {
     while (getFilePosition() <= end) {
       newSize = in.readLine(value, maxLineLength,
       newSize = in.readLine(value, maxLineLength,
           Math.max(maxBytesToConsume(pos), maxLineLength));
           Math.max(maxBytesToConsume(pos), maxLineLength));
-      if (newSize == 0) {
-        break;
-      }
       pos += newSize;
       pos += newSize;
       if (newSize < maxLineLength) {
       if (newSize < maxLineLength) {
         break;
         break;
@@ -219,4 +216,4 @@ public class LineRecordReader extends RecordReader<LongWritable, Text> {
       }
       }
     }
     }
   }
   }
-}
+}