1
0

ExternalMapperReducer.java 1.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748
  1. import java.io.IOException;
  2. import java.util.Iterator;
  3. import org.apache.hadoop.io.IntWritable;
  4. import org.apache.hadoop.io.Text;
  5. import org.apache.hadoop.io.Writable;
  6. import org.apache.hadoop.io.WritableComparable;
  7. import org.apache.hadoop.mapred.JobConf;
  8. import org.apache.hadoop.mapred.Mapper;
  9. import org.apache.hadoop.mapred.OutputCollector;
  10. import org.apache.hadoop.mapred.Reducer;
  11. import org.apache.hadoop.mapred.Reporter;
  12. public class ExternalMapperReducer
  13. implements Mapper, Reducer {
  14. public void configure(JobConf job) {
  15. }
  16. public void close()
  17. throws IOException {
  18. }
  19. public void map(WritableComparable key, Writable value,
  20. OutputCollector output, Reporter reporter)
  21. throws IOException {
  22. if (value instanceof Text) {
  23. Text text = (Text)value;
  24. ExternalWritable ext = new ExternalWritable(text.toString());
  25. output.collect(ext, new IntWritable(1));
  26. }
  27. }
  28. public void reduce(WritableComparable key, Iterator values,
  29. OutputCollector output, Reporter reporter)
  30. throws IOException {
  31. int count = 0;
  32. while (values.hasNext()) {
  33. count++;
  34. values.next();
  35. }
  36. output.collect(key, new IntWritable(count));
  37. }
  38. }