소스 검색

HADOOP-4947. Make Chukwa command parsing more forgiving of whitespace. Contributed by Ari Rabkin.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/branches/branch-0.20@732656 13f79535-47bb-0310-9956-ffa450edef68
Christopher Douglas 16 년 전
부모
커밋
6f7bb27b12

+ 6 - 3
CHANGES.txt

@@ -295,6 +295,9 @@ Release 0.20.0 - Unreleased
     HADOOP-4980. Improve code layout of capacity scheduler to make it 
     easier to fix some blocker bugs. (Vivek Ratan via yhemanth)
 
+    HADOOP-4916. Make user/location of Chukwa installation configurable by an
+    external properties file. (Eric Yang via cdouglas)
+
   OPTIMIZATIONS
 
     HADOOP-3293. Fixes FileInputFormat to do provide locations for splits
@@ -520,12 +523,12 @@ Release 0.20.0 - Unreleased
     HADOOP-4884. Make tool tip date format match standard HICC format. (Eric
     Yang via cdouglas)
 
-    HADOOP-4916. Make user/location of Chukwa installation configurable by an
-    external properties file. (Eric Yang via cdouglas)
-
     HADOOP-4925. Make Chukwa sender properties configurable. (Ari Rabkin via
     cdouglas)
 
+    HADOOP-4947. Make Chukwa command parsing more forgiving of whitespace. (Ari
+    Rabkin via cdouglas)
+
 Release 0.19.1 - Unreleased
 
   IMPROVEMENTS

+ 8 - 5
src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/adaptor/filetailer/FileTailingAdaptor.java

@@ -29,6 +29,8 @@ import org.apache.log4j.Logger;
 
 import java.io.*;
 import java.util.Timer;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * An adaptor that repeatedly tails a specified file, sending the new bytes.
@@ -81,11 +83,12 @@ public class FileTailingAdaptor implements Adaptor
 	    this.type = type;
 	    this.dest = dest;
 	    this.attempts = 0;
-			  
-	    String[] words = params.split(" ");
-	    if(words.length > 1) {
-	        offsetOfFirstByte = Long.parseLong(words[0]);
-	        toWatch = new File(params.substring(words[0].length() + 1));
+			
+	    Pattern cmd = Pattern.compile("(\\d+)\\s+(.+)");
+	    Matcher m = cmd.matcher(params);
+	    if(m.matches()) {
+	        offsetOfFirstByte = Long.parseLong(m.group(1));
+	        toWatch = new File(m.group(2));
 	    } else {
 	        toWatch = new File(params);
 	    }

+ 1 - 1
src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/AgentControlSocketListener.java

@@ -80,7 +80,7 @@ public class AgentControlSocketListener extends Thread {
      * @throws IOException
      */
     public void processCommand(String cmd, PrintStream out) throws IOException  {
-      String[] words = cmd.split(" ");
+      String[] words = cmd.split("\\s+");
       if (log.isDebugEnabled())
   		{ log.debug("command from " + connection.getRemoteSocketAddress() + ":"+ cmd);}
       

+ 28 - 40
src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java

@@ -30,6 +30,8 @@ import org.apache.log4j.Logger;
 
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.io.*;
 
 /**
@@ -231,59 +233,45 @@ public class ChukwaAgent
     }
 
   }
-
+  // words should contain (space delimited):
+  // 0) command ("add")
+  // 1) AdaptorClassname
+  // 2) dataType (e.g. "hadoop_log")
+  // 3) params <optional>
+  // (e.g. for files, this is filename,
+  // but can be arbitrarily many space
+  // delimited agent specific params )
+  // 4) offset
+  Pattern addCmdPattern = Pattern.compile("add\\s+(\\S+)\\s+(\\S+)\\s+(.*\\S)?\\s*(\\d+)\\s*");
   // FIXME: should handle bad lines here
   public long processCommand(String cmd)
   {
-    String[] words = cmd.split(" ");
-    if (words[0].equalsIgnoreCase("add"))
-    {
-      // words should contain (space delimited):
-      // 0) command ("add")
-      // 1) AdaptorClassname
-      // 2) dataType (e.g. "hadoop_log")
-      // 3) params <optional>
-      // (e.g. for files, this is filename,
-      // but can be arbitrarily many space
-      // delimited agent specific params )
-      // 4) offset
-
-      long offset;
-      try
-      {
-        offset = Long.parseLong(words[words.length - 1]);
-      } catch (NumberFormatException e)
-      {
+    Matcher m = addCmdPattern.matcher(cmd);
+    if (m.matches()) {
+      long offset;  //check for obvious errors first
+      try {
+        offset = Long.parseLong(m.group(4));
+      } catch (NumberFormatException e)  {
         log.warn("malformed line " + cmd);
         return -1L;
       }
-      String adaptorName = words[1];
+      
+      String adaptorName = m.group(1);
+      String dataType = m.group(2);
+      String params = m.group(3);
+      if(params == null)
+        params = "";
 
       Adaptor adaptor = AdaptorFactory.createAdaptor(adaptorName);
-      if (adaptor == null)
-      {
+      if (adaptor == null)  {
         log.warn("Error creating adaptor from adaptor name " + adaptorName);
         return -1L;
       }
 
-      String dataType = words[2];
-      String streamName = "";
-      String params = "";
-      if (words.length > 4)
-      { // no argument
-        int begParams = adaptorName.length() + dataType.length() + 6;
-        // length("ADD x type ") = length(x) + 5, i.e. count letters & spaces
-        params = cmd.substring(begParams, cmd.length()
-            - words[words.length - 1].length() - 1);
-        streamName = params.substring(params.indexOf(" ") + 1, params.length());
-      }
       long adaptorID;
-      synchronized (adaptorsByNumber)
-      {
-        for (Map.Entry<Long, Adaptor> a : adaptorsByNumber.entrySet())
-        {
-          if (streamName.intern() == a.getValue().getStreamName().intern())
-          {
+      synchronized (adaptorsByNumber) {
+        for (Map.Entry<Long, Adaptor> a : adaptorsByNumber.entrySet()) {
+          if (params.intern() == a.getValue().getStreamName().intern()) {
             log.warn(params + " already exist, skipping.");
             return -1;
           }

+ 1 - 1
src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgent.java

@@ -65,7 +65,7 @@ public class TestAgent extends TestCase {
         ArrayList<Long> runningAdaptors = new ArrayList<Long>();
        
         for(int i = 1; i < 7; ++i) {
-          long l = agent.processCommand("add org.apache.hadoop.chukwa.util.ConstRateAdaptor raw"+i+ " 2000"+i+" 0");
+          long l = agent.processCommand("add  org.apache.hadoop.chukwa.util.ConstRateAdaptor  raw"+i+ " 2000"+i+" 0");
           assertTrue(l != -1);
           runningAdaptors.add(l);
         }