HADOOP-2403. Escapes some special characters before logging to history files. Contributed by Amareshwari Sriramadasu.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@696641 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -619,6 +619,9 @@ Trunk (unreleased changes)
     HADOOP-4195. Close compressor before returning to codec pool.
     (acmurthy via omalley)
 
+    HADOOP-2403. Escapes some special characters before logging to 
+    history files. (Amareshwari Sriramadasu via ddas)
+
 Release 0.18.1 - 2008-09-17
 
   IMPROVEMENTS

src/mapred/org/apache/hadoop/mapred/JobHistory.java

@@ -68,8 +68,12 @@ public class JobHistory {
   public static final Log LOG = LogFactory.getLog(JobHistory.class);
   private static final String DELIMITER = " ";
   private static final String LINE_DELIMITER = ".";
+  private static final char LINE_DELIMITER_CHAR = '.';
+  private static final char[] charsToEscape = new char[] {'"', '=', 
+                                                LINE_DELIMITER_CHAR};
   private static final String KEY = "(\\w+)";
-  private static final String VALUE = "[[^\"]?]+"; // anything but a " in ""
+  // value is any character other than quote, but escaped quotes can be there
+  private static final String VALUE = "[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*"; 
   
   private static final Pattern pattern = Pattern.compile(KEY + "=" + "\"" + VALUE + "\"");
   
@@ -177,7 +181,10 @@ public class JobHistory {
       StringBuffer buf = new StringBuffer(); 
       while ((line = reader.readLine())!= null){
         buf.append(line); 
-        if (!line.trim().endsWith("\"" + DELIMITER + LINE_DELIMITER)){
+        if (!line.trim().endsWith(LINE_DELIMITER) || 
+            line.trim().endsWith(StringUtils.escapeString(LINE_DELIMITER,
+                          StringUtils.ESCAPE_CHAR, LINE_DELIMITER_CHAR))) {
+          buf.append("\n");
           continue; 
         }
         parseLine(buf.toString(), l);
@@ -204,9 +211,11 @@ public class JobHistory {
 
     while(matcher.find()){
       String tuple = matcher.group(0);
-      String []parts = tuple.split("=");
-      
-      parseBuffer.put(Keys.valueOf(parts[0]), parts[1].substring(1, parts[1].length() -1));
+      String []parts = StringUtils.split(tuple, StringUtils.ESCAPE_CHAR, '=');
+      String value = parts[1].substring(1, parts[1].length() -1);
+      value = StringUtils.unEscapeString(value, StringUtils.ESCAPE_CHAR,
+                                         charsToEscape);
+      parseBuffer.put(Keys.valueOf(parts[0]), value);
     }
 
     l.handle(RecordTypes.valueOf(recType), parseBuffer); 
@@ -224,6 +233,8 @@ public class JobHistory {
   
   static void log(PrintWriter out, RecordTypes recordType, Keys key, 
                   String value){
+    value = StringUtils.escapeString(value, StringUtils.ESCAPE_CHAR, 
+                                     charsToEscape);
     out.println(recordType.name() + DELIMITER + key + "=\"" + value + "\""
                 + DELIMITER + LINE_DELIMITER); 
   }
@@ -243,6 +254,8 @@ public class JobHistory {
     for(int i =0; i< keys.length; i++){
       buf.append(keys[i]);
       buf.append("=\"");
+      values[i] = StringUtils.escapeString(values[i],
+                                StringUtils.ESCAPE_CHAR, charsToEscape);
       buf.append(values[i]);
       buf.append("\"");
       buf.append(DELIMITER); 

src/test/org/apache/hadoop/mapred/TestJobHistoryParsing.java

@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobHistory.*;
+
+import junit.framework.TestCase;
+
+public class TestJobHistoryParsing  extends TestCase {
+  ArrayList<PrintWriter> historyWriter = new ArrayList<PrintWriter>();
+
+  /**
+   * Listener for a test history log file, it populates JobHistory.JobInfo 
+   * object with data from log file. 
+   */
+  static class TestListener implements Listener {
+    JobInfo job;
+
+    TestListener(JobInfo job) {
+      this.job = job;
+    }
+    // JobHistory.Listener implementation 
+    public void handle(RecordTypes recType, 
+                       Map<JobHistory.Keys, String> values)
+    throws IOException {
+      if (recType == JobHistory.RecordTypes.Job) {
+        job.handle(values);
+      }
+    }
+  }
+
+  public void testHistoryParsing() throws IOException {
+    // open a test history file
+    Path historyDir = new Path(System.getProperty("test.build.data", "."), 
+                                "history");
+    FileSystem fs = FileSystem.getLocal(new JobConf());
+    if (!fs.mkdirs(historyDir)) {
+      fail("Failed to create history directory");
+    }
+    Path historyLog = new Path(historyDir, "testlog");
+    PrintWriter out = new PrintWriter(fs.create(historyLog));
+    historyWriter.add(out);
+    // log keys and values into history
+    String value1 = "Value has equal=to, \"quotes\" and spaces in it";
+    String value2 = "Value has \n new line \n and " + 
+                    "dot followed by new line .\n in it ";
+    String value3 = "Value has characters: " +
+                    "`1234567890-=qwertyuiop[]\\asdfghjkl;'zxcvbnm,./" +
+                    "~!@#$%^&*()_+QWERTYUIOP{}|ASDFGHJKL:\"'ZXCVBNM<>?" + 
+                    "\t\b\n\f\"\n in it";
+    String value4 = "Value ends with escape\\";
+    String value5 = "Value ends with \\\" \\.\n";
+    JobHistory.log(historyWriter, RecordTypes.Job, 
+                   new JobHistory.Keys[] {Keys.JOBTRACKERID, 
+                                          Keys.TRACKER_NAME, 
+                                          Keys.JOBNAME, 
+                                          Keys.JOBCONF,
+                                          Keys.USER},
+                   new String[] {value1, value2, value3, value4, value5});
+    // close history file
+    out.close();
+    historyWriter.remove(out);
+
+    // parse history
+    String jobId = "job_200809171136_0001"; // random jobid for tesing 
+    JobHistory.JobInfo job = new JobHistory.JobInfo(jobId);
+    JobHistory.parseHistoryFromFS(historyLog.toString(), 
+                 new TestListener(job), fs);
+    // validate keys and values
+    assertEquals(value1, job.get(Keys.JOBTRACKERID));
+    assertEquals(value2, job.get(Keys.TRACKER_NAME));
+    assertEquals(value3, job.get(Keys.JOBNAME));
+    assertEquals(value4, job.get(Keys.JOBCONF));
+    assertEquals(value5, job.get(Keys.USER));
+  }
+}