|
@@ -27,13 +27,15 @@ import org.apache.hadoop.conf.Configuration;
|
|
|
import org.apache.hadoop.fs.FSDataInputStream;
|
|
|
import org.apache.hadoop.fs.FileSystem;
|
|
|
import org.apache.hadoop.fs.Path;
|
|
|
+import org.apache.hadoop.io.SequenceFile.CompressionType;
|
|
|
import org.apache.hadoop.test.GenericTestUtils;
|
|
|
import org.junit.Test;
|
|
|
|
|
|
+/** Tests sync based seek reads/write intervals inside SequenceFiles. */
|
|
|
public class TestSequenceFileSync {
|
|
|
private static final int NUMRECORDS = 2000;
|
|
|
private static final int RECORDSIZE = 80;
|
|
|
- private static final Random rand = new Random();
|
|
|
+ private static final Random RAND = new Random();
|
|
|
|
|
|
private final static String REC_FMT = "%d RECORDID %d : ";
|
|
|
|
|
@@ -46,37 +48,110 @@ public class TestSequenceFileSync {
|
|
|
reader.next(key, val);
|
|
|
assertEquals(key.get(), expectedRecord);
|
|
|
final String test = String.format(REC_FMT, expectedRecord, expectedRecord);
|
|
|
- assertEquals("Invalid value " + val, 0, val.find(test, 0));
|
|
|
+ assertEquals(
|
|
|
+ "Invalid value in iter " + iter + ": " + val,
|
|
|
+ 0,
|
|
|
+ val.find(test, 0));
|
|
|
+ }
|
|
|
+
|
|
|
+ @Test
|
|
|
+ public void testDefaultSyncInterval() throws IOException {
|
|
|
+ // Uses the default sync interval of 100 KB
|
|
|
+ final Configuration conf = new Configuration();
|
|
|
+ final FileSystem fs = FileSystem.getLocal(conf);
|
|
|
+ final Path path = new Path(GenericTestUtils.getTempPath(
|
|
|
+ "sequencefile.sync.test"));
|
|
|
+ final IntWritable input = new IntWritable();
|
|
|
+ final Text val = new Text();
|
|
|
+ SequenceFile.Writer writer = new SequenceFile.Writer(
|
|
|
+ conf,
|
|
|
+ SequenceFile.Writer.file(path),
|
|
|
+ SequenceFile.Writer.compression(CompressionType.NONE),
|
|
|
+ SequenceFile.Writer.keyClass(IntWritable.class),
|
|
|
+ SequenceFile.Writer.valueClass(Text.class)
|
|
|
+ );
|
|
|
+ try {
|
|
|
+ writeSequenceFile(writer, NUMRECORDS*4);
|
|
|
+ for (int i = 0; i < 5; i++) {
|
|
|
+ final SequenceFile.Reader reader;
|
|
|
+
|
|
|
+ //try different SequenceFile.Reader constructors
|
|
|
+ if (i % 2 == 0) {
|
|
|
+ final int buffersize = conf.getInt("io.file.buffer.size", 4096);
|
|
|
+ reader = new SequenceFile.Reader(conf,
|
|
|
+ SequenceFile.Reader.file(path),
|
|
|
+ SequenceFile.Reader.bufferSize(buffersize));
|
|
|
+ } else {
|
|
|
+ final FSDataInputStream in = fs.open(path);
|
|
|
+ final long length = fs.getFileStatus(path).getLen();
|
|
|
+ reader = new SequenceFile.Reader(conf,
|
|
|
+ SequenceFile.Reader.stream(in),
|
|
|
+ SequenceFile.Reader.start(0L),
|
|
|
+ SequenceFile.Reader.length(length));
|
|
|
+ }
|
|
|
+
|
|
|
+ try {
|
|
|
+ forOffset(reader, input, val, i, 0, 0);
|
|
|
+ forOffset(reader, input, val, i, 65, 0);
|
|
|
+ // There would be over 1000 records within
|
|
|
+ // this sync interval
|
|
|
+ forOffset(reader, input, val, i, 2000, 1101);
|
|
|
+ forOffset(reader, input, val, i, 0, 0);
|
|
|
+ } finally {
|
|
|
+ reader.close();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } finally {
|
|
|
+ fs.delete(path, false);
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
@Test
|
|
|
public void testLowSyncpoint() throws IOException {
|
|
|
+ // Uses a smaller sync interval of 2000 bytes
|
|
|
final Configuration conf = new Configuration();
|
|
|
final FileSystem fs = FileSystem.getLocal(conf);
|
|
|
final Path path = new Path(GenericTestUtils.getTempPath(
|
|
|
"sequencefile.sync.test"));
|
|
|
final IntWritable input = new IntWritable();
|
|
|
final Text val = new Text();
|
|
|
- SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path,
|
|
|
- IntWritable.class, Text.class);
|
|
|
+ SequenceFile.Writer writer = new SequenceFile.Writer(
|
|
|
+ conf,
|
|
|
+ SequenceFile.Writer.file(path),
|
|
|
+ SequenceFile.Writer.compression(CompressionType.NONE),
|
|
|
+ SequenceFile.Writer.keyClass(IntWritable.class),
|
|
|
+ SequenceFile.Writer.valueClass(Text.class),
|
|
|
+ SequenceFile.Writer.syncInterval(20*100)
|
|
|
+ );
|
|
|
+ // Ensure the custom sync interval value is set
|
|
|
+ assertEquals(writer.syncInterval, 20*100);
|
|
|
try {
|
|
|
writeSequenceFile(writer, NUMRECORDS);
|
|
|
- for (int i = 0; i < 5 ; i++) {
|
|
|
- final SequenceFile.Reader reader;
|
|
|
+ for (int i = 0; i < 5; i++) {
|
|
|
+ final SequenceFile.Reader reader;
|
|
|
|
|
|
- //try different SequenceFile.Reader constructors
|
|
|
- if (i % 2 == 0) {
|
|
|
- reader = new SequenceFile.Reader(fs, path, conf);
|
|
|
- } else {
|
|
|
- final FSDataInputStream in = fs.open(path);
|
|
|
- final long length = fs.getFileStatus(path).getLen();
|
|
|
- final int buffersize = conf.getInt("io.file.buffer.size", 4096);
|
|
|
- reader = new SequenceFile.Reader(in, buffersize, 0L, length, conf);
|
|
|
- }
|
|
|
+ //try different SequenceFile.Reader constructors
|
|
|
+ if (i % 2 == 0) {
|
|
|
+ final int bufferSize = conf.getInt("io.file.buffer.size", 4096);
|
|
|
+ reader = new SequenceFile.Reader(
|
|
|
+ conf,
|
|
|
+ SequenceFile.Reader.file(path),
|
|
|
+ SequenceFile.Reader.bufferSize(bufferSize));
|
|
|
+ } else {
|
|
|
+ final FSDataInputStream in = fs.open(path);
|
|
|
+ final long length = fs.getFileStatus(path).getLen();
|
|
|
+ reader = new SequenceFile.Reader(
|
|
|
+ conf,
|
|
|
+ SequenceFile.Reader.stream(in),
|
|
|
+ SequenceFile.Reader.start(0L),
|
|
|
+ SequenceFile.Reader.length(length));
|
|
|
+ }
|
|
|
|
|
|
- try {
|
|
|
+ try {
|
|
|
forOffset(reader, input, val, i, 0, 0);
|
|
|
forOffset(reader, input, val, i, 65, 0);
|
|
|
+ // There would be only a few records within
|
|
|
+ // this sync interval
|
|
|
forOffset(reader, input, val, i, 2000, 21);
|
|
|
forOffset(reader, input, val, i, 0, 0);
|
|
|
} finally {
|
|
@@ -88,7 +163,7 @@ public class TestSequenceFileSync {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- public static void writeSequenceFile(SequenceFile.Writer writer,
|
|
|
+ private static void writeSequenceFile(SequenceFile.Writer writer,
|
|
|
int numRecords) throws IOException {
|
|
|
final IntWritable key = new IntWritable();
|
|
|
final Text val = new Text();
|
|
@@ -100,13 +175,13 @@ public class TestSequenceFileSync {
|
|
|
writer.close();
|
|
|
}
|
|
|
|
|
|
- static void randomText(Text val, int id, int recordSize) {
|
|
|
+ private static void randomText(Text val, int id, int recordSize) {
|
|
|
val.clear();
|
|
|
final StringBuilder ret = new StringBuilder(recordSize);
|
|
|
ret.append(String.format(REC_FMT, id, id));
|
|
|
recordSize -= ret.length();
|
|
|
for (int i = 0; i < recordSize; ++i) {
|
|
|
- ret.append(rand.nextInt(9));
|
|
|
+ ret.append(RAND.nextInt(9));
|
|
|
}
|
|
|
val.set(ret.toString());
|
|
|
}
|