HADOOP-5042. Add a cleanup stage to log rollover in Chukwa appender. Contributed by Jerome Boulon.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@747547 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -131,6 +131,9 @@ Trunk (unreleased changes)
 
     HADOOP-4191. Adds a testcase for JobHistory. (Ravi Gummadi via ddas)
 
+    HADOOP-5042. Add a cleanup stage to log rollover in Chukwa appender.
+    (Jerome Boulon via cdouglas)
+
   OPTIMIZATIONS
 
   BUG FIXES

src/contrib/chukwa/conf/system-data-loader.properties

@@ -2,6 +2,8 @@ log4j.rootLogger=INFO, R
 log4j.appender.R=org.apache.hadoop.chukwa.inputtools.log4j.ChukwaDailyRollingFileAppender
 log4j.appender.R.File=${CHUKWA_LOG_DIR}/${RECORD_TYPE}.log
 log4j.appender.R.recordType=${RECORD_TYPE}
+log4j.appender.R.maxBackupIndex=3
+log4j.appender.R.cleanUpRegex=$fileName.20*
 log4j.appender.R.layout=org.apache.log4j.PatternLayout
 log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
 

src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java

@@ -10,13 +10,18 @@
 package org.apache.hadoop.chukwa.inputtools.log4j;
 
 import java.io.File;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Calendar;
+import java.util.Collections;
 import java.util.Date;
 import java.util.GregorianCalendar;
+import java.util.List;
 import java.util.Locale;
 import java.util.TimeZone;
+import java.util.regex.Pattern;
 
 import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
 import org.apache.hadoop.chukwa.util.RecordConstants;
@@ -130,8 +135,7 @@ import org.apache.log4j.spi.LoggingEvent;
     <b>DatePattern</b> option. The text before the colon is interpeted
     as the protocol specificaion of a URL which is probably not what
     you want. */
-
-
+    
 public class ChukwaDailyRollingFileAppender extends FileAppender {
 
 	static Logger log = Logger.getLogger(ChukwaDailyRollingFileAppender.class);
@@ -172,6 +176,16 @@ public class ChukwaDailyRollingFileAppender extends FileAppender {
     The next time we estimate a rollover should occur. */
   private long nextCheck = System.currentTimeMillis () - 1;
 
+  /**
+   * Regex to select log files to be deleted
+   */
+  private String cleanUpRegex = null;
+  
+  /**
+   * Set the maximum number of backup files to keep around.
+   */
+  private int maxBackupIndex = 10;
+  
   Date now = new Date();
 
   SimpleDateFormat sdf;
@@ -351,6 +365,7 @@ public class ChukwaDailyRollingFileAppender extends FileAppender {
     }
 
     File file = new File(fileName);
+    
     boolean result = file.renameTo(target);
     if(result) {
       LogLog.debug(fileName +" -> "+ scheduledFilename);
@@ -367,8 +382,82 @@ public class ChukwaDailyRollingFileAppender extends FileAppender {
       errorHandler.error("setFile("+fileName+", false) call failed.");
     }    
     scheduledFilename = datedFilename;
+    cleanUp();
+  }
+
+  
+  public String getCleanUpRegex() {
+    return cleanUpRegex;
+  }
+
+  public void setCleanUpRegex(String cleanUpRegex) {
+    this.cleanUpRegex = cleanUpRegex;
+  }
+
+  public int getMaxBackupIndex() {
+    return maxBackupIndex;
+  }
+
+  public void setMaxBackupIndex(int maxBackupIndex) {
+    this.maxBackupIndex = maxBackupIndex;
   }
 
+  protected synchronized void cleanUp() {
+    String regex = "";
+    try {
+      File actualFile = new File(fileName);
+      
+      String directoryName = actualFile.getParent();
+      String actualFileName = actualFile.getName();
+      File dirList = new File(directoryName);
+      
+      
+      if (cleanUpRegex == null || !cleanUpRegex.contains("$fileName")) {
+        LogLog.error("cleanUpRegex == null || !cleanUpRegex.contains(\"$fileName\")");
+        return;
+      }
+      regex =cleanUpRegex.replace("$fileName", actualFileName);
+      String[] dirFiles = dirList.list(new LogFilter(actualFileName,regex));
+      
+      List<String> files = new ArrayList<String>();
+      for(String file: dirFiles) {
+        files.add(file); 
+      }
+      Collections.sort(files);
+      
+      while(files.size() > maxBackupIndex) {
+        String file = files.remove(0);
+        File f = new File(directoryName + "/" +file);
+        f.delete();
+        LogLog.debug("Removing: " +file);
+      }
+    } catch(Exception e) {
+      errorHandler.error("cleanUp("+fileName+"," + regex +") call failed.");
+    }
+  }
+  
+  private class LogFilter implements FilenameFilter {
+    private Pattern p = null;
+    private String logFile = null;
+ 
+    public LogFilter(String logFile,String regex) {
+      this.logFile = logFile;
+      p = Pattern.compile(regex); 
+    }
+ 
+    @Override
+    public boolean accept(File dir, String name) {
+      // ignore current log file
+      if (name.intern() == this.logFile.intern() ) {
+        return false;
+      }
+      //ignore file without the same prefix
+      if (!name.startsWith(logFile)) {
+        return false;
+      }
+      return p.matcher(name).find();
+    }
+  }
   
   private class ClientFinalizer extends Thread 
   {
@@ -614,3 +703,4 @@ int type = ChukwaDailyRollingFileAppender.TOP_OF_TROUBLE;
     return getTime();
   }
 }
+