|
|
@@ -20,12 +20,14 @@ package org.apache.hadoop.util;
|
|
|
import java.io.FileNotFoundException;
|
|
|
import java.io.FileOutputStream;
|
|
|
import java.io.PrintStream;
|
|
|
+import java.lang.reflect.Constructor;
|
|
|
import java.util.ArrayList;
|
|
|
import java.util.List;
|
|
|
import java.util.Properties;
|
|
|
import java.util.Random;
|
|
|
import java.util.zip.CRC32;
|
|
|
import java.util.zip.Checksum;
|
|
|
+
|
|
|
import org.junit.Assert;
|
|
|
import org.junit.Test;
|
|
|
|
|
|
@@ -187,12 +189,18 @@ public class TestPureJavaCrc32 {
|
|
|
long polynomial = Long.parseLong(args[0], 16);
|
|
|
|
|
|
int i = 8;
|
|
|
- final PrintStream out = new PrintStream(
|
|
|
- new FileOutputStream("table" + i + ".txt"), true);
|
|
|
final Table t = new Table(i, 16, polynomial);
|
|
|
final String s = t.toString();
|
|
|
System.out.println(s);
|
|
|
- out.println(s);
|
|
|
+
|
|
|
+ //print to a file
|
|
|
+ final PrintStream out = new PrintStream(
|
|
|
+ new FileOutputStream("table" + i + ".txt"), true);
|
|
|
+ try {
|
|
|
+ out.println(s);
|
|
|
+ } finally {
|
|
|
+ out.close();
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@@ -210,10 +218,15 @@ public class TestPureJavaCrc32 {
|
|
|
public static final int MAX_LEN = 32*1024*1024; // up to 32MB chunks
|
|
|
public static final int BYTES_PER_SIZE = MAX_LEN * 4;
|
|
|
|
|
|
- static final Checksum zip = new CRC32();
|
|
|
- static final Checksum[] CRCS = {new PureJavaCrc32()};
|
|
|
+ static final Class<? extends Checksum> zip = CRC32.class;
|
|
|
+ static final List<Class<? extends Checksum>> CRCS = new ArrayList<Class<? extends Checksum>>();
|
|
|
+ static {
|
|
|
+ CRCS.add(zip);
|
|
|
+ CRCS.add(PureJavaCrc32.class);
|
|
|
+ }
|
|
|
+
|
|
|
|
|
|
- public static void main(String args[]) {
|
|
|
+ public static void main(String args[]) throws Exception {
|
|
|
printSystemProperties(System.out);
|
|
|
doBench(CRCS, System.out);
|
|
|
}
|
|
|
@@ -223,76 +236,140 @@ public class TestPureJavaCrc32 {
|
|
|
out.printf(" %" + w + "s |", s);
|
|
|
}
|
|
|
|
|
|
- private static void doBench(final Checksum[] crcs, final PrintStream out) {
|
|
|
- final ArrayList<Checksum> a = new ArrayList<Checksum>();
|
|
|
- a.add(zip);
|
|
|
- for (Checksum c : crcs)
|
|
|
- if(c.getClass() != zip.getClass())
|
|
|
- a.add(c);
|
|
|
- doBench(a, out);
|
|
|
- }
|
|
|
-
|
|
|
- private static void doBench(final List<Checksum> crcs, final PrintStream out
|
|
|
- ) {
|
|
|
+ private static void doBench(final List<Class<? extends Checksum>> crcs,
|
|
|
+ final PrintStream out) throws Exception {
|
|
|
final byte[] bytes = new byte[MAX_LEN];
|
|
|
new Random().nextBytes(bytes);
|
|
|
|
|
|
// Print header
|
|
|
- out.printf("\nPerformance Table (The unit is MB/sec)\n||");
|
|
|
- final String title = "Num Bytes";
|
|
|
- printCell("Num Bytes", 0, out);
|
|
|
- for (Checksum c : crcs) {
|
|
|
- out.printf("|");
|
|
|
- printCell(c.getClass().getSimpleName(), 8, out);
|
|
|
- }
|
|
|
- out.printf("|\n");
|
|
|
+ out.printf("\nPerformance Table (The unit is MB/sec; #T = #Theads)\n");
|
|
|
|
|
|
// Warm up implementations to get jit going.
|
|
|
- for (Checksum c : crcs) {
|
|
|
- doBench(c, bytes, 2, null);
|
|
|
- doBench(c, bytes, 2101, null);
|
|
|
+ for (Class<? extends Checksum> c : crcs) {
|
|
|
+ doBench(c, 1, bytes, 2);
|
|
|
+ doBench(c, 1, bytes, 2101);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Test on a variety of sizes with different number of threads
|
|
|
+ for (int size = 32; size <= MAX_LEN; size <<= 1) {
|
|
|
+ doBench(crcs, bytes, size, out);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void doBench(final List<Class<? extends Checksum>> crcs,
|
|
|
+ final byte[] bytes, final int size, final PrintStream out) throws Exception {
|
|
|
+ final String numBytesStr = " #Bytes ";
|
|
|
+ final String numThreadsStr = "#T";
|
|
|
+ final String diffStr = "% diff";
|
|
|
+
|
|
|
+ out.print('|');
|
|
|
+ printCell(numBytesStr, 0, out);
|
|
|
+ printCell(numThreadsStr, 0, out);
|
|
|
+ for (int i = 0; i < crcs.size(); i++) {
|
|
|
+ final Class<? extends Checksum> c = crcs.get(i);
|
|
|
+ out.print('|');
|
|
|
+ printCell(c.getSimpleName(), 8, out);
|
|
|
+ for(int j = 0; j < i; j++) {
|
|
|
+ printCell(diffStr, diffStr.length(), out);
|
|
|
+ }
|
|
|
}
|
|
|
+ out.printf("\n");
|
|
|
|
|
|
- // Test on a variety of sizes
|
|
|
- for (int size = 1; size < MAX_LEN; size *= 2) {
|
|
|
+ for(int numThreads = 1; numThreads <= 16; numThreads <<= 1) {
|
|
|
out.printf("|");
|
|
|
- printCell(String.valueOf(size), title.length()+1, out);
|
|
|
+ printCell(String.valueOf(size), numBytesStr.length(), out);
|
|
|
+ printCell(String.valueOf(numThreads), numThreadsStr.length(), out);
|
|
|
|
|
|
- Long expected = null;
|
|
|
- for(Checksum c : crcs) {
|
|
|
+ BenchResult expected = null;
|
|
|
+ final List<BenchResult> previous = new ArrayList<BenchResult>();
|
|
|
+ for(Class<? extends Checksum> c : crcs) {
|
|
|
System.gc();
|
|
|
- final long result = doBench(c, bytes, size, out);
|
|
|
- if(c.getClass() == zip.getClass()) {
|
|
|
+
|
|
|
+ final BenchResult result = doBench(c, numThreads, bytes, size);
|
|
|
+ printCell(String.format("%9.1f", result.mbps),
|
|
|
+ c.getSimpleName().length()+1, out);
|
|
|
+
|
|
|
+ //check result
|
|
|
+ if(c == zip) {
|
|
|
expected = result;
|
|
|
- } else if (result != expected) {
|
|
|
- throw new RuntimeException(c.getClass() + " has bugs!");
|
|
|
+ } else if (expected == null) {
|
|
|
+ throw new RuntimeException("The first class is "
|
|
|
+ + c.getName() + " but not " + zip.getName());
|
|
|
+ } else if (result.value != expected.value) {
|
|
|
+ throw new RuntimeException(c + " has bugs!");
|
|
|
+ }
|
|
|
+
|
|
|
+ //compare result with previous
|
|
|
+ for(BenchResult p : previous) {
|
|
|
+ final double diff = (result.mbps - p.mbps) / p.mbps * 100;
|
|
|
+ printCell(String.format("%5.1f%%", diff), diffStr.length(), out);
|
|
|
}
|
|
|
-
|
|
|
+ previous.add(result);
|
|
|
}
|
|
|
out.printf("\n");
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- private static long doBench(Checksum crc, byte[] bytes, int size,
|
|
|
- PrintStream out) {
|
|
|
- final String name = crc.getClass().getSimpleName();
|
|
|
- final int trials = BYTES_PER_SIZE / size;
|
|
|
+ private static BenchResult doBench(Class<? extends Checksum> clazz,
|
|
|
+ final int numThreads, final byte[] bytes, final int size)
|
|
|
+ throws Exception {
|
|
|
+
|
|
|
+ final Thread[] threads = new Thread[numThreads];
|
|
|
+ final BenchResult[] results = new BenchResult[threads.length];
|
|
|
+
|
|
|
+ {
|
|
|
+ final int trials = BYTES_PER_SIZE / size;
|
|
|
+ final double mbProcessed = trials * size / 1024.0 / 1024.0;
|
|
|
+ final Constructor<? extends Checksum> ctor = clazz.getConstructor();
|
|
|
+
|
|
|
+ for(int i = 0; i < threads.length; i++) {
|
|
|
+ final int index = i;
|
|
|
+ threads[i] = new Thread() {
|
|
|
+ final Checksum crc = ctor.newInstance();
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void run() {
|
|
|
+ final long st = System.nanoTime();
|
|
|
+ crc.reset();
|
|
|
+ for (int i = 0; i < trials; i++) {
|
|
|
+ crc.update(bytes, 0, size);
|
|
|
+ }
|
|
|
+ final long et = System.nanoTime();
|
|
|
+ double secsElapsed = (et - st) / 1000000000.0d;
|
|
|
+ results[index] = new BenchResult(crc.getValue(), mbProcessed/secsElapsed);
|
|
|
+ }
|
|
|
+ };
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ for(int i = 0; i < threads.length; i++) {
|
|
|
+ threads[i].start();
|
|
|
+ }
|
|
|
+ for(int i = 0; i < threads.length; i++) {
|
|
|
+ threads[i].join();
|
|
|
+ }
|
|
|
|
|
|
- final long st = System.nanoTime();
|
|
|
- crc.reset();
|
|
|
- for (int i = 0; i < trials; i++) {
|
|
|
- crc.update(bytes, 0, size);
|
|
|
+ final long expected = results[0].value;
|
|
|
+ double sum = results[0].mbps;
|
|
|
+ for(int i = 1; i < results.length; i++) {
|
|
|
+ if (results[i].value != expected) {
|
|
|
+ throw new AssertionError(clazz.getSimpleName() + " results not matched.");
|
|
|
+ }
|
|
|
+ sum += results[i].mbps;
|
|
|
}
|
|
|
- final long result = crc.getValue();
|
|
|
- final long et = System.nanoTime();
|
|
|
-
|
|
|
- double mbProcessed = trials * size / 1024.0 / 1024.0;
|
|
|
- double secsElapsed = (et - st) / 1000000000.0d;
|
|
|
- if (out != null) {
|
|
|
- final String s = String.format("%9.3f", mbProcessed/secsElapsed);
|
|
|
- printCell(s, name.length()+1, out);
|
|
|
+ return new BenchResult(expected, sum/results.length);
|
|
|
+ }
|
|
|
+
|
|
|
+ private static class BenchResult {
|
|
|
+ /** CRC value */
|
|
|
+ final long value;
|
|
|
+ /** Speed (MB per second) */
|
|
|
+ final double mbps;
|
|
|
+
|
|
|
+ BenchResult(long value, double mbps) {
|
|
|
+ this.value = value;
|
|
|
+ this.mbps = mbps;
|
|
|
}
|
|
|
- return result;
|
|
|
}
|
|
|
|
|
|
private static void printSystemProperties(PrintStream out) {
|
Tsz-wo Sze