HADOOP-5697. Change org.apache.hadoop.examples.Grep to use new mapreduce api. Contributed by Amareshwari Sriramadasu.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@767055 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -243,6 +243,9 @@ Trunk (unreleased changes)
     HADOOP-5681. Change examples RandomWriter and RandomTextWriter to 
     use new mapreduce API. (Amareshwari Sriramadasu via sharad)
 
+    HADOOP-5697. Change org.apache.hadoop.examples.Grep to use new 
+    mapreduce api. (Amareshwari Sriramadasu via sharad)
+
   OPTIMIZATIONS
 
     HADOOP-5595. NameNode does not need to run a replicator to choose a

src/examples/org/apache/hadoop/examples/Grep.java

@@ -25,8 +25,14 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapred.lib.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.map.InverseMapper;
+import org.apache.hadoop.mapreduce.lib.map.RegexMapper;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
@@ -38,14 +44,19 @@ public class Grep extends Configured implements Tool {
     if (args.length < 3) {
       System.out.println("Grep <inDir> <outDir> <regex> [<group>]");
       ToolRunner.printGenericCommandUsage(System.out);
-      return -1;
+      return 2;
     }
 
     Path tempDir =
       new Path("grep-temp-"+
           Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
 
-    JobConf grepJob = new JobConf(getConf(), Grep.class);
+    Configuration conf = getConf();
+    conf.set("mapred.mapper.regex", args[2]);
+    if (args.length == 4)
+      conf.set("mapred.mapper.regex.group", args[3]);
+
+    Job grepJob = new Job(conf);
     
     try {
       
@@ -54,37 +65,34 @@ public class Grep extends Configured implements Tool {
       FileInputFormat.setInputPaths(grepJob, args[0]);
 
       grepJob.setMapperClass(RegexMapper.class);
-      grepJob.set("mapred.mapper.regex", args[2]);
-      if (args.length == 4)
-        grepJob.set("mapred.mapper.regex.group", args[3]);
 
       grepJob.setCombinerClass(LongSumReducer.class);
       grepJob.setReducerClass(LongSumReducer.class);
 
       FileOutputFormat.setOutputPath(grepJob, tempDir);
-      grepJob.setOutputFormat(SequenceFileOutputFormat.class);
+      grepJob.setOutputFormatClass(SequenceFileOutputFormat.class);
       grepJob.setOutputKeyClass(Text.class);
       grepJob.setOutputValueClass(LongWritable.class);
 
-      JobClient.runJob(grepJob);
+      grepJob.waitForCompletion(true);
 
-      JobConf sortJob = new JobConf(Grep.class);
+      Job sortJob = new Job(conf);
       sortJob.setJobName("grep-sort");
 
       FileInputFormat.setInputPaths(sortJob, tempDir);
-      sortJob.setInputFormat(SequenceFileInputFormat.class);
+      sortJob.setInputFormatClass(SequenceFileInputFormat.class);
 
       sortJob.setMapperClass(InverseMapper.class);
 
       sortJob.setNumReduceTasks(1);                 // write a single file
       FileOutputFormat.setOutputPath(sortJob, new Path(args[1]));
-      sortJob.setOutputKeyComparatorClass           // sort by decreasing freq
-      (LongWritable.DecreasingComparator.class);
+      sortJob.setSortComparatorClass(          // sort by decreasing freq
+        LongWritable.DecreasingComparator.class);
 
-      JobClient.runJob(sortJob);
+      sortJob.waitForCompletion(true);
     }
     finally {
-      FileSystem.get(grepJob).delete(tempDir, true);
+      FileSystem.get(conf).delete(tempDir, true);
     }
     return 0;
   }

src/mapred/org/apache/hadoop/mapred/lib/RegexMapper.java

@@ -31,7 +31,10 @@ import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
 
 
-/** A {@link Mapper} that extracts text matching a regular expression. */
+/** A {@link Mapper} that extracts text matching a regular expression.
+ * @deprecated Use {@link org.apache.hadoop.mapreduce.lib.map.RegexMapper}
+ */
+@Deprecated
 public class RegexMapper<K> extends MapReduceBase
     implements Mapper<K, Text, Text, LongWritable> {
 

src/mapred/org/apache/hadoop/mapreduce/lib/map/RegexMapper.java

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.map;
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+
+
+/** A {@link Mapper} that extracts text matching a regular expression. */
+public class RegexMapper<K> extends Mapper<K, Text, Text, LongWritable> {
+
+  private Pattern pattern;
+  private int group;
+
+  public void setup(Context context) {
+    Configuration conf = context.getConfiguration();
+    pattern = Pattern.compile(conf.get("mapred.mapper.regex"));
+    group = conf.getInt("mapred.mapper.regex.group", 0);
+  }
+
+  public void map(K key, Text value,
+                  Context context)
+    throws IOException, InterruptedException {
+    String text = value.toString();
+    Matcher matcher = pattern.matcher(text);
+    while (matcher.find()) {
+      context.write(new Text(matcher.group(group)), new LongWritable(1));
+    }
+  }
+}