HADOOP-3970. Provides a way to recover counters written to JobHistory. Contributed by Amar Kamat.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@693385 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -138,6 +138,9 @@ Trunk (unreleased changes)
     HADOOP-3581. Prevents memory intensive user tasks from taking down 
     nodes. (Vinod K V via ddas)
 
+    HADOOP-3970. Provides a way to recover counters written to JobHistory.
+    (Amar Kamat via ddas)
+
   IMPROVEMENTS
 
     HADOOP-3908. Fuse-dfs: better error message if llibhdfs.so doesn't exist.

src/core/org/apache/hadoop/util/StringUtils.java

@@ -327,16 +327,11 @@ public class StringUtils {
     }
     ArrayList<String> strList = new ArrayList<String>();
     StringBuilder split = new StringBuilder();
-    int numPreEscapes = 0;
-    for (int i=0; i<str.length(); i++) {
-      char curChar = str.charAt(i);
-      if (numPreEscapes==0 && curChar == separator) { // separator 
-        strList.add(split.toString());
-        split.setLength(0); // clear the split
-      } else {
-        split.append(curChar);
-        numPreEscapes = (curChar == escapeChar)?(++numPreEscapes)%2:0;
-      }
+    int index = 0;
+    while ((index = findNext(str, separator, escapeChar, index, split)) >= 0) {
+      ++index; // move over the separator for next search
+      strList.add(split.toString());
+      split.setLength(0); // reset the buffer 
     }
     strList.add(split.toString());
     // remove trailing empty split(s)
@@ -347,6 +342,33 @@ public class StringUtils {
     return strList.toArray(new String[strList.size()]);
   }
   
+  /**
+   * Finds the first occurrence of the separator character ignoring the escaped
+   * separators starting from the index. Note the substring between the index
+   * and the position of the separator is passed.
+   * @param str the source string
+   * @param separator the character to find
+   * @param escapeChar character used to escape
+   * @param start from where to search
+   * @param split used to pass back the extracted string
+   */
+  public static int findNext(String str, char separator, char escapeChar, 
+                             int start, StringBuilder split) {
+    int numPreEscapes = 0;
+    for (int i = start; i < str.length(); i++) {
+      char curChar = str.charAt(i);
+      if (numPreEscapes == 0 && curChar == separator) { // separator 
+        return i;
+      } else {
+        split.append(curChar);
+        numPreEscapes = (curChar == escapeChar)
+                        ? (++numPreEscapes) % 2
+                        : 0;
+      }
+    }
+    return -1;
+  }
+  
   /**
    * Escape commas in the string using the default escape char
    * @param str a string
@@ -367,13 +389,31 @@ public class StringUtils {
    */
   public static String escapeString(
       String str, char escapeChar, char charToEscape) {
+    return escapeString(str, escapeChar, new char[] {charToEscape});
+  }
+  
+  // check if the character array has the character 
+  private static boolean hasChar(char[] chars, char character) {
+    for (char target : chars) {
+      if (character == target) {
+        return true;
+      }
+    }
+    return false;
+  }
+  
+  /**
+   * @param charsToEscape array of characters to be escaped
+   */
+  public static String escapeString(String str, char escapeChar, 
+                                    char[] charsToEscape) {
     if (str == null) {
       return null;
     }
     StringBuilder result = new StringBuilder();
     for (int i=0; i<str.length(); i++) {
       char curChar = str.charAt(i);
-      if (curChar == escapeChar || curChar == charToEscape) {
+      if (curChar == escapeChar || hasChar(charsToEscape, curChar)) {
         // special char
         result.append(escapeChar);
       }
@@ -402,6 +442,14 @@ public class StringUtils {
    */
   public static String unEscapeString(
       String str, char escapeChar, char charToEscape) {
+    return unEscapeString(str, escapeChar, new char[] {charToEscape});
+  }
+  
+  /**
+   * @param charsToEscape array of characters to unescape
+   */
+  public static String unEscapeString(String str, char escapeChar, 
+                                      char[] charsToEscape) {
     if (str == null) {
       return null;
     }
@@ -410,7 +458,7 @@ public class StringUtils {
     for (int i=0; i<str.length(); i++) {
       char curChar = str.charAt(i);
       if (hasPreEscape) {
-        if (curChar != escapeChar && curChar != charToEscape) {
+        if (curChar != escapeChar && !hasChar(charsToEscape, curChar)) {
           // no special char
           throw new IllegalArgumentException("Illegal escaped string " + str + 
               " unescaped " + escapeChar + " at " + (i-1));
@@ -419,9 +467,9 @@ public class StringUtils {
         result.append(curChar);
         hasPreEscape = false;
       } else {
-        if (curChar == charToEscape) {
+        if (hasChar(charsToEscape, curChar)) {
           throw new IllegalArgumentException("Illegal escaped string " + str + 
-              " unescaped " + charToEscape + " at " + i);
+              " unescaped " + curChar + " at " + i);
         } else if (curChar == escapeChar) {
           hasPreEscape = true;
         } else {
@@ -431,7 +479,7 @@ public class StringUtils {
     }
     if (hasPreEscape ) {
       throw new IllegalArgumentException("Illegal escaped string " + str + 
-          ", not expecting " + charToEscape + " in the end." );
+          ", not expecting " + escapeChar + " in the end." );
     }
     return result.toString();
   }

src/mapred/org/apache/hadoop/mapred/Counters.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.mapred;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -29,11 +30,14 @@ import java.util.Iterator;
 import java.util.Map;
 import java.util.MissingResourceException;
 import java.util.ResourceBundle;
+import java.util.Set;
 
 import org.apache.commons.logging.*;
+import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * A set of named counters.
@@ -47,6 +51,15 @@ import org.apache.hadoop.io.WritableUtils;
  */
 public class Counters implements Writable, Iterable<Counters.Group> {
   private static final Log LOG = LogFactory.getLog(Counters.class);
+  private static final char GROUP_OPEN = '{';
+  private static final char GROUP_CLOSE = '}';
+  private static final char COUNTER_OPEN = '[';
+  private static final char COUNTER_CLOSE = ']';
+  private static final char UNIT_OPEN = '(';
+  private static final char UNIT_CLOSE = ')';
+  private static char[] charsToEscape =  {GROUP_OPEN, GROUP_CLOSE, 
+                                          COUNTER_OPEN, COUNTER_CLOSE, 
+                                          UNIT_OPEN, UNIT_CLOSE};
   
   //private static Log log = LogFactory.getLog("Counters.class");
   
@@ -111,6 +124,41 @@ public class Counters implements Writable, Iterable<Counters.Group> {
       return displayName;
     }
     
+    /**
+     * Set the display name of the counter.
+     */
+    public synchronized void setDisplayName(String displayName) {
+      this.displayName = displayName;
+    }
+    
+    /**
+     * Returns the compact stringified version of the counter in the format
+     * [(actual-name)(display-name)(value)]
+     */
+    public synchronized String makeEscapedCompactString() {
+      StringBuffer buf = new StringBuffer();
+      buf.append(COUNTER_OPEN);
+      
+      // Add the counter name
+      buf.append(UNIT_OPEN);
+      buf.append(escape(getName()));
+      buf.append(UNIT_CLOSE);
+      
+      // Add the display name
+      buf.append(UNIT_OPEN);
+      buf.append(escape(getDisplayName()));
+      buf.append(UNIT_CLOSE);
+      
+      // Add the value
+      buf.append(UNIT_OPEN);
+      buf.append(this.value);
+      buf.append(UNIT_CLOSE);
+      
+      buf.append(COUNTER_CLOSE);
+      
+      return buf.toString();
+    }
+    
     /**
      * What is the current value of this counter?
      * @return the current value
@@ -180,6 +228,48 @@ public class Counters implements Writable, Iterable<Counters.Group> {
       return displayName;
     }
     
+    /**
+     * Set the display name
+     */
+    public void setDisplayName(String displayName) {
+      this.displayName = displayName;
+    }
+    
+    /**
+     * Returns the compact stringified version of the group in the format
+     * {(actual-name)(display-name)(value)[][][]} where [] are compact strings for the
+     * counters within.
+     */
+    public String makeEscapedCompactString() {
+      StringBuffer buf = new StringBuffer();
+      buf.append(GROUP_OPEN); // group start
+      
+      // Add the group name
+      buf.append(UNIT_OPEN);
+      buf.append(escape(getName()));
+      buf.append(UNIT_CLOSE);
+      
+      // Add the display name
+      buf.append(UNIT_OPEN);
+      buf.append(escape(getDisplayName()));
+      buf.append(UNIT_CLOSE);
+      
+      // write the value
+      for(Counter counter: subcounters.values()) {
+        buf.append(counter.makeEscapedCompactString());
+      }
+      
+      buf.append(GROUP_CLOSE); // group end
+      return buf.toString();
+    }
+        
+    /**
+     * Returns the names of the counters within
+     */
+    public synchronized Set<String> getCounterNames() {
+      return subcounters.keySet();
+    }
+    
     /**
      * Returns the value of the specified counter, or 0 if the counter does
      * not exist.
@@ -499,6 +589,125 @@ public class Counters implements Writable, Iterable<Counters.Group> {
     return buffer.toString();
   }
   
+  /**
+   * Represent the counter in a textual format that can be converted back to 
+   * its object form
+   * @return the string in the following format
+   * {(groupname)(group-displayname)[(countername)(displayname)(value)][][]}{}{}
+   */
+  public synchronized String makeEscapedCompactString() {
+    StringBuffer buffer = new StringBuffer();
+    for(Group group: this){
+      buffer.append(group.makeEscapedCompactString());
+    }
+    return buffer.toString();
+  }
+
+  // Extracts a block (data enclosed within delimeters) ignoring escape 
+  // sequences. Throws ParseException if an incomplete block is found else 
+  // returns null.
+  private static String getBlock(String str, char open, char close, 
+                                IntWritable index) throws ParseException {
+    StringBuilder split = new StringBuilder();
+    int next = StringUtils.findNext(str, open, StringUtils.ESCAPE_CHAR, 
+                                    index.get(), split);
+    split.setLength(0); // clear the buffer
+    if (next >= 0) {
+      ++next; // move over '('
+      
+      next = StringUtils.findNext(str, close, StringUtils.ESCAPE_CHAR, 
+                                   next, split);
+      if (next >= 0) {
+        ++next; // move over ')'
+        index.set(next);
+        return split.toString(); // found a block
+      } else {
+        throw new ParseException("Unexpected end of block", next);
+      }
+    }
+    return null; // found nothing
+  }
+  
+  /**
+   * Convert a stringified counter representation into a counter object. Note 
+   * that the counter can be recovered if its stringified using 
+   * {@link #makeEscapedCompactString()}. 
+   * @return a Counter
+   */
+  public static Counters fromEscapedCompactString(String compactString) 
+  throws ParseException {
+    Counters counters = new Counters();
+    IntWritable index = new IntWritable(0);
+    
+    // Get the group to work on
+    String groupString = 
+      getBlock(compactString, GROUP_OPEN, GROUP_CLOSE, index);
+    
+    while (groupString != null) {
+      IntWritable groupIndex = new IntWritable(0);
+      
+      // Get the actual name
+      String groupName = 
+        getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex);
+      groupName = unescape(groupName);
+      
+      // Get the display name
+      String groupDisplayName = 
+        getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex);
+      groupDisplayName = unescape(groupDisplayName);
+      
+      // Get the counters
+      Group group = counters.getGroup(groupName);
+      group.setDisplayName(groupDisplayName);
+      
+      String counterString = 
+        getBlock(groupString, COUNTER_OPEN, COUNTER_CLOSE, groupIndex);
+      
+      while (counterString != null) {
+        IntWritable counterIndex = new IntWritable(0);
+        
+        // Get the actual name
+        String counterName = 
+          getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex);
+        counterName = unescape(counterName);
+        
+        // Get the display name
+        String counterDisplayName = 
+          getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, counterIndex);
+        counterDisplayName = unescape(counterDisplayName);
+        
+        // Get the value
+        long value = 
+          Long.parseLong(getBlock(counterString, UNIT_OPEN, UNIT_CLOSE, 
+                                  counterIndex));
+        
+        // Add the counter
+        Counter counter = group.getCounterForName(counterName);
+        counter.setDisplayName(counterDisplayName);
+        counter.increment(value);
+        
+        // Get the next counter
+        counterString = 
+          getBlock(groupString, COUNTER_OPEN, COUNTER_CLOSE, groupIndex);
+      }
+      
+      groupString = getBlock(compactString, GROUP_OPEN, GROUP_CLOSE, index);
+    }
+    return counters;
+  }
+
+  // Escapes all the delimiters for counters i.e {,[,(,),],}
+  private static String escape(String string) {
+    return StringUtils.escapeString(string, StringUtils.ESCAPE_CHAR, 
+                                    charsToEscape);
+  }
+  
+  // Unescapes all the delimiters for counters i.e {,[,(,),],}
+  private static String unescape(String string) {
+    return StringUtils.unEscapeString(string, StringUtils.ESCAPE_CHAR, 
+                                      charsToEscape);
+  }
+  
   public static class Application {
     //special counters which are written by the application and are 
     //used by the framework.