HADOOP-1813 [hbase] OOME makes zombie of region server

Forgot to add below as part of last commit.


git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@575950 13f79535-47bb-0310-9956-ffa450edef68

src/contrib/hbase/src/java/org/apache/hadoop/hbase/Chore.java

@@ -0,0 +1,85 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.util.Sleeper;
+import org.apache.hadoop.hbase.util.Threads;
+
+/**
+ * Chore is a task performed on a period in hbase.  The chore is run in its own
+ * thread. This base abstract class provides while loop and sleeping facility.
+ * If an unhandled exception, the threads exit is logged.
+ * Implementers just need to add checking if there is work to be done and if
+ * so, do it.  Its the base of most of the chore threads in hbase.
+ */
+public abstract class Chore extends Thread {
+  private final Log LOG = LogFactory.getLog(this.getClass());
+  private final Sleeper sleeper;
+  protected final AtomicBoolean stop;
+  
+  /**
+   * @param p Period at which we should run.  Will be adjusted appropriately
+   * should we find work and it takes time to complete.
+   * @param s When this flag is set to true, this thread will cleanup and exit
+   * cleanly.
+   */
+  public Chore(final int p, final AtomicBoolean s) {
+    super();
+    this.sleeper = new Sleeper(p, s);
+    this.stop = s;
+  }
+
+  public void run() {
+    try {
+      initialChore();
+      this.sleeper.sleep();
+      while(!this.stop.get()) {
+        long startTime = System.currentTimeMillis();
+        chore();
+        this.sleeper.sleep(startTime);
+      }
+    } finally {
+      LOG.info(getName() + " exiting");
+    }
+  }
+  
+  /**
+   * Override to run a task before we start looping.
+   */
+  protected void initialChore() {
+    // Default does nothing.
+  }
+  
+  /**
+   * Look for chores.  If any found, do them else just return.
+   */
+  protected abstract void chore();
+
+  /**
+   * Sleep for period.
+   */
+  protected void sleep() {
+    this.sleeper.sleep();
+  }
+}

src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Sleeper.java

@@ -0,0 +1,68 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Sleeper for current thread.
+ * Sleeps for passed period.  Also checks passed boolean and if interrupted,
+ * will return if the flag is set (rather than go back to sleep until its 
+ * sleep time is up).
+ */
+public class Sleeper {
+  private final int period;
+  private AtomicBoolean stop;
+  
+  public Sleeper(final int sleep, final AtomicBoolean stop) {
+    this.period = sleep;
+    this.stop = stop;
+  }
+  
+  /**
+   * Sleep for period.
+   */
+  public void sleep() {
+    sleep(System.currentTimeMillis());
+  }
+  
+  /**
+   * Sleep for period adjusted by passed <code>startTime<code>
+   * @param startTime Time some task started previous to now.  Time to sleep
+   * will be docked current time minus passed <code>startTime<code>.
+   */
+  public void sleep(final long startTime) {
+    if (this.stop.get()) {
+      return;
+    }
+    long waitTime = this.period - (System.currentTimeMillis() - startTime);
+    if (waitTime > 0) {
+      try {
+        Thread.sleep(waitTime);
+      } catch(InterruptedException iex) {
+        // We we interrupted because we're meant to stop?  If not, just
+        // continue ignoring the interruption
+        if (this.stop.get()) {
+          return;
+        }
+      }
+    }
+  }
+}

src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Threads.java

@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.lang.Thread.UncaughtExceptionHandler;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Thread Utility
+ */
+public class Threads {
+  protected static final Log LOG = LogFactory.getLog(Threads.class);
+  
+  /**
+   * Utility method that sets name, daemon status and starts passed thread.
+   * @param t
+   * @param name
+   * @return Returns the passed Thread <code>t</code>.
+   */
+  public static Thread setDaemonThreadRunning(final Thread t,
+      final String name) {
+    return setDaemonThreadRunning(t, name, null);
+  }
+    
+    /**
+     * Utility method that sets name, daemon status and starts passed thread.
+     * @param t
+     * @param name
+     * @param handler A handler to set on the thread.  Pass null if want to
+     * use default handler.
+     * @return Returns the passed Thread <code>t</code>.
+     */
+    public static Thread setDaemonThreadRunning(final Thread t,
+        final String name, final UncaughtExceptionHandler handler) {
+    t.setName(name);
+    if (handler != null) {
+      t.setUncaughtExceptionHandler(handler);
+    }
+    t.setDaemon(true);
+    t.start();
+    return t;
+  }
+}

src/contrib/hbase/src/test/org/apache/hadoop/hbase/OOMEHMaster.java

@@ -0,0 +1,61 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * An HMaster that runs out of memory.
+ * Everytime a region server reports in, add to the retained heap of memory.
+ * Needs to be started manually as in
+ * <code>${HBASE_HOME}/bin/hbase ./bin/hbase org.apache.hadoop.hbase.OOMEHMaster start/code>.
+ */
+public class OOMEHMaster extends HMaster {
+  private List<byte []> retainer = new ArrayList<byte[]>();
+  
+  public OOMEHMaster(Configuration conf) throws IOException {
+    super(conf);
+  }
+
+  public OOMEHMaster(Path dir, HServerAddress address, Configuration conf)
+      throws IOException {
+    super(dir, address, conf);
+  }
+  
+  @Override
+  public HMsg[] regionServerReport(HServerInfo serverInfo, HMsg[] msgs)
+  throws IOException {
+    // Retain 1M.
+    this.retainer.add(new byte [1024 * 1024]);
+    return super.regionServerReport(serverInfo, msgs);
+  }
+
+  /**
+   * @param args
+   */
+  public static void main(String[] args) {
+    doMain(args, OOMEHMaster.class);
+  }
+}

src/contrib/hbase/src/test/org/apache/hadoop/hbase/OOMERegionServer.java

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.io.Text;
+
+/**
+ * A region server that will OOME.
+ * Everytime {@link #batchUpdate(Text, long, BatchUpdate)} is called, we add
+ * keep around a reference to the batch.  Use this class to test OOME extremes.
+ * Needs to be started manually as in
+ * <code>${HBASE_HOME}/bin/hbase ./bin/hbase org.apache.hadoop.hbase.OOMERegionServer start</code>.
+ */
+public class OOMERegionServer extends HRegionServer {
+  private List<BatchUpdate> retainer = new ArrayList<BatchUpdate>();
+
+  public OOMERegionServer(Configuration conf) throws IOException {
+    super(conf);
+  }
+
+  public OOMERegionServer(Path rootDir, HServerAddress address,
+      Configuration conf) throws IOException {
+    super(rootDir, address, conf);
+  }
+  
+  @Override
+  public void batchUpdate(Text regionName, long timestamp, BatchUpdate b)
+      throws IOException {
+    super.batchUpdate(regionName, timestamp, b);
+    for (int i = 0; i < 30; i++) {
+      // Add the batch update 30 times to bring on the OOME faster.
+      this.retainer.add(b);
+    }
+  }
+  
+  public static void main(String[] args) {
+    HRegionServer.doMain(args, OOMERegionServer.class);
+  }
+}