|
@@ -136,32 +136,47 @@ public class TestKeyValueTextInputFormat extends TestCase {
|
|
}
|
|
}
|
|
|
|
|
|
public void testUTF8() throws Exception {
|
|
public void testUTF8() throws Exception {
|
|
- LineReader in = makeStream("abcd\u20acbdcd\u20ac");
|
|
|
|
- Text line = new Text();
|
|
|
|
- in.readLine(line);
|
|
|
|
- assertEquals("readLine changed utf8 characters",
|
|
|
|
- "abcd\u20acbdcd\u20ac", line.toString());
|
|
|
|
- in = makeStream("abc\u200axyz");
|
|
|
|
- in.readLine(line);
|
|
|
|
- assertEquals("split on fake newline", "abc\u200axyz", line.toString());
|
|
|
|
|
|
+ LineReader in = null;
|
|
|
|
+
|
|
|
|
+ try {
|
|
|
|
+ in = makeStream("abcd\u20acbdcd\u20ac");
|
|
|
|
+ Text line = new Text();
|
|
|
|
+ in.readLine(line);
|
|
|
|
+ assertEquals("readLine changed utf8 characters",
|
|
|
|
+ "abcd\u20acbdcd\u20ac", line.toString());
|
|
|
|
+ in = makeStream("abc\u200axyz");
|
|
|
|
+ in.readLine(line);
|
|
|
|
+ assertEquals("split on fake newline", "abc\u200axyz", line.toString());
|
|
|
|
+ } finally {
|
|
|
|
+ if (in != null) {
|
|
|
|
+ in.close();
|
|
|
|
+ }
|
|
|
|
+ }
|
|
}
|
|
}
|
|
|
|
|
|
public void testNewLines() throws Exception {
|
|
public void testNewLines() throws Exception {
|
|
- LineReader in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
|
|
|
|
- Text out = new Text();
|
|
|
|
- in.readLine(out);
|
|
|
|
- assertEquals("line1 length", 1, out.getLength());
|
|
|
|
- in.readLine(out);
|
|
|
|
- assertEquals("line2 length", 2, out.getLength());
|
|
|
|
- in.readLine(out);
|
|
|
|
- assertEquals("line3 length", 0, out.getLength());
|
|
|
|
- in.readLine(out);
|
|
|
|
- assertEquals("line4 length", 3, out.getLength());
|
|
|
|
- in.readLine(out);
|
|
|
|
- assertEquals("line5 length", 4, out.getLength());
|
|
|
|
- in.readLine(out);
|
|
|
|
- assertEquals("line5 length", 5, out.getLength());
|
|
|
|
- assertEquals("end of file", 0, in.readLine(out));
|
|
|
|
|
|
+ LineReader in = null;
|
|
|
|
+ try {
|
|
|
|
+ in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
|
|
|
|
+ Text out = new Text();
|
|
|
|
+ in.readLine(out);
|
|
|
|
+ assertEquals("line1 length", 1, out.getLength());
|
|
|
|
+ in.readLine(out);
|
|
|
|
+ assertEquals("line2 length", 2, out.getLength());
|
|
|
|
+ in.readLine(out);
|
|
|
|
+ assertEquals("line3 length", 0, out.getLength());
|
|
|
|
+ in.readLine(out);
|
|
|
|
+ assertEquals("line4 length", 3, out.getLength());
|
|
|
|
+ in.readLine(out);
|
|
|
|
+ assertEquals("line5 length", 4, out.getLength());
|
|
|
|
+ in.readLine(out);
|
|
|
|
+ assertEquals("line5 length", 5, out.getLength());
|
|
|
|
+ assertEquals("end of file", 0, in.readLine(out));
|
|
|
|
+ } finally {
|
|
|
|
+ if (in != null) {
|
|
|
|
+ in.close();
|
|
|
|
+ }
|
|
|
|
+ }
|
|
}
|
|
}
|
|
|
|
|
|
private static void writeFile(FileSystem fs, Path name,
|
|
private static void writeFile(FileSystem fs, Path name,
|
|
@@ -183,14 +198,21 @@ public class TestKeyValueTextInputFormat extends TestCase {
|
|
InputSplit split,
|
|
InputSplit split,
|
|
JobConf job) throws IOException {
|
|
JobConf job) throws IOException {
|
|
List<Text> result = new ArrayList<Text>();
|
|
List<Text> result = new ArrayList<Text>();
|
|
- RecordReader<Text, Text> reader = format.getRecordReader(split, job,
|
|
|
|
- voidReporter);
|
|
|
|
- Text key = reader.createKey();
|
|
|
|
- Text value = reader.createValue();
|
|
|
|
- while (reader.next(key, value)) {
|
|
|
|
- result.add(value);
|
|
|
|
- value = reader.createValue();
|
|
|
|
- }
|
|
|
|
|
|
+ RecordReader<Text, Text> reader = null;
|
|
|
|
+
|
|
|
|
+ try {
|
|
|
|
+ reader = format.getRecordReader(split, job, voidReporter);
|
|
|
|
+ Text key = reader.createKey();
|
|
|
|
+ Text value = reader.createValue();
|
|
|
|
+ while (reader.next(key, value)) {
|
|
|
|
+ result.add(value);
|
|
|
|
+ value = (Text) reader.createValue();
|
|
|
|
+ }
|
|
|
|
+ } finally {
|
|
|
|
+ if (reader != null) {
|
|
|
|
+ reader.close();
|
|
|
|
+ }
|
|
|
|
+ }
|
|
return result;
|
|
return result;
|
|
}
|
|
}
|
|
|
|
|