|
@@ -28,6 +28,7 @@ import java.util.LinkedList;
|
|
import java.util.List;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Map;
|
|
import java.util.Set;
|
|
import java.util.Set;
|
|
|
|
+import java.util.concurrent.TimeUnit;
|
|
|
|
|
|
import org.apache.commons.logging.Log;
|
|
import org.apache.commons.logging.Log;
|
|
import org.apache.commons.logging.LogFactory;
|
|
import org.apache.commons.logging.LogFactory;
|
|
@@ -45,9 +46,9 @@ import org.apache.hadoop.net.NetworkTopology;
|
|
import org.apache.hadoop.net.Node;
|
|
import org.apache.hadoop.net.Node;
|
|
import org.apache.hadoop.net.NodeBase;
|
|
import org.apache.hadoop.net.NodeBase;
|
|
import org.apache.hadoop.util.ReflectionUtils;
|
|
import org.apache.hadoop.util.ReflectionUtils;
|
|
|
|
+import org.apache.hadoop.util.StopWatch;
|
|
import org.apache.hadoop.util.StringUtils;
|
|
import org.apache.hadoop.util.StringUtils;
|
|
|
|
|
|
-import com.google.common.base.Stopwatch;
|
|
|
|
import com.google.common.collect.Iterables;
|
|
import com.google.common.collect.Iterables;
|
|
|
|
|
|
/**
|
|
/**
|
|
@@ -223,7 +224,7 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> {
|
|
org.apache.hadoop.mapreduce.lib.input.FileInputFormat.LIST_STATUS_NUM_THREADS,
|
|
org.apache.hadoop.mapreduce.lib.input.FileInputFormat.LIST_STATUS_NUM_THREADS,
|
|
org.apache.hadoop.mapreduce.lib.input.FileInputFormat.DEFAULT_LIST_STATUS_NUM_THREADS);
|
|
org.apache.hadoop.mapreduce.lib.input.FileInputFormat.DEFAULT_LIST_STATUS_NUM_THREADS);
|
|
|
|
|
|
- Stopwatch sw = new Stopwatch().start();
|
|
|
|
|
|
+ StopWatch sw = new StopWatch().start();
|
|
if (numThreads == 1) {
|
|
if (numThreads == 1) {
|
|
List<FileStatus> locatedFiles = singleThreadedListStatus(job, dirs, inputFilter, recursive);
|
|
List<FileStatus> locatedFiles = singleThreadedListStatus(job, dirs, inputFilter, recursive);
|
|
result = locatedFiles.toArray(new FileStatus[locatedFiles.size()]);
|
|
result = locatedFiles.toArray(new FileStatus[locatedFiles.size()]);
|
|
@@ -242,7 +243,8 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> {
|
|
|
|
|
|
sw.stop();
|
|
sw.stop();
|
|
if (LOG.isDebugEnabled()) {
|
|
if (LOG.isDebugEnabled()) {
|
|
- LOG.debug("Time taken to get FileStatuses: " + sw.elapsedMillis());
|
|
|
|
|
|
+ LOG.debug("Time taken to get FileStatuses: "
|
|
|
|
+ + sw.now(TimeUnit.MILLISECONDS));
|
|
}
|
|
}
|
|
LOG.info("Total input paths to process : " + result.length);
|
|
LOG.info("Total input paths to process : " + result.length);
|
|
return result;
|
|
return result;
|
|
@@ -309,7 +311,7 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> {
|
|
* they're too big.*/
|
|
* they're too big.*/
|
|
public InputSplit[] getSplits(JobConf job, int numSplits)
|
|
public InputSplit[] getSplits(JobConf job, int numSplits)
|
|
throws IOException {
|
|
throws IOException {
|
|
- Stopwatch sw = new Stopwatch().start();
|
|
|
|
|
|
+ StopWatch sw = new StopWatch().start();
|
|
FileStatus[] files = listStatus(job);
|
|
FileStatus[] files = listStatus(job);
|
|
|
|
|
|
// Save the number of input files for metrics/loadgen
|
|
// Save the number of input files for metrics/loadgen
|
|
@@ -371,7 +373,7 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> {
|
|
sw.stop();
|
|
sw.stop();
|
|
if (LOG.isDebugEnabled()) {
|
|
if (LOG.isDebugEnabled()) {
|
|
LOG.debug("Total # of splits generated by getSplits: " + splits.size()
|
|
LOG.debug("Total # of splits generated by getSplits: " + splits.size()
|
|
- + ", TimeTaken: " + sw.elapsedMillis());
|
|
|
|
|
|
+ + ", TimeTaken: " + sw.now(TimeUnit.MILLISECONDS));
|
|
}
|
|
}
|
|
return splits.toArray(new FileSplit[splits.size()]);
|
|
return splits.toArray(new FileSplit[splits.size()]);
|
|
}
|
|
}
|