ZOOKEEPER-2185: Run server with -XX:+HeapDumpOnOutOfMemoryError and

-XX:OnOutOfMemoryError='kill %p' (Chris Nauroth via rgs)

git-svn-id: https://svn.apache.org/repos/asf/zookeeper/trunk@1686296 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -208,6 +208,9 @@ IMPROVEMENTS:
   ZOOKEEPER-2214: Findbugs warning: LearnerHandler.packetToString Dead
   store to local variable (Hongchao Deng via rgs)
 
+  ZOOKEEPER-2185: Run server with -XX:+HeapDumpOnOutOfMemoryError and
+  -XX:OnOutOfMemoryError='kill %p' (Chris Nauroth via rgs)
+
 Release 3.5.0 - 8/4/2014
 
 NEW FEATURES:

bin/zkServer.cmd

@@ -21,6 +21,6 @@ set ZOOMAIN=org.apache.zookeeper.server.quorum.QuorumPeerMain
 set ZOO_LOG_FILE=zookeeper-%USERNAME%-server-%COMPUTERNAME%.log
 
 echo on
-call %JAVA% "-Dzookeeper.log.dir=%ZOO_LOG_DIR%" "-Dzookeeper.root.logger=%ZOO_LOG4J_PROP%" "-Dzookeeper.log.file=%ZOO_LOG_FILE%" -cp "%CLASSPATH%" %ZOOMAIN% "%ZOOCFG%" %*
+call %JAVA% "-Dzookeeper.log.dir=%ZOO_LOG_DIR%" "-Dzookeeper.root.logger=%ZOO_LOG4J_PROP%" "-Dzookeeper.log.file=%ZOO_LOG_FILE%" "-XX:+HeapDumpOnOutOfMemoryError" "-XX:OnOutOfMemoryError=cmd /c taskkill /pid %%%%p /t /f" -cp "%CLASSPATH%" %ZOOMAIN% "%ZOOCFG%" %*
 
 endlocal

bin/zkServer.sh

@@ -145,8 +145,9 @@ start)
          exit 0
       fi
     fi
-    nohup "$JAVA" $ZOO_DATADIR_AUTOCREATE "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \
-    "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" \
+    nohup "$JAVA" $ZOO_DATADIR_AUTOCREATE "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" \
+    "-Dzookeeper.log.file=${ZOO_LOG_FILE}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" \
+    -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryError='kill -9 %p' \
     -cp "$CLASSPATH" $JVMFLAGS $ZOOMAIN "$ZOOCFG" > "$_ZOO_DAEMON_OUT" 2>&1 < /dev/null &
     if [ $? -eq 0 ]
     then
@@ -174,12 +175,16 @@ start-foreground)
     if [ "${ZOO_NOEXEC}" != "" ]; then
       ZOO_CMD=("$JAVA")
     fi
-    "${ZOO_CMD[@]}" $ZOO_DATADIR_AUTOCREATE "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \
-    "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" \
+    "${ZOO_CMD[@]}" $ZOO_DATADIR_AUTOCREATE "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" \
+    "-Dzookeeper.log.file=${ZOO_LOG_FILE}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" \
+    -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryError='kill -9 %p' \
     -cp "$CLASSPATH" $JVMFLAGS $ZOOMAIN "$ZOOCFG"
     ;;
 print-cmd)
-    echo "\"$JAVA\" $ZOO_DATADIR_AUTOCREATE -Dzookeeper.log.dir=\"${ZOO_LOG_DIR}\" -Dzookeeper.root.logger=\"${ZOO_LOG4J_PROP}\" -Dzookeeper.log.file=\"${ZOO_LOG_FILE}\" -cp \"$CLASSPATH\" $JVMFLAGS $ZOOMAIN \"$ZOOCFG\" > \"$_ZOO_DAEMON_OUT\" 2>&1 < /dev/null"
+    echo "\"$JAVA\" $ZOO_DATADIR_AUTOCREATE -Dzookeeper.log.dir=\"${ZOO_LOG_DIR}\" \
+    -Dzookeeper.log.file=\"${ZOO_LOG_FILE}\" -Dzookeeper.root.logger=\"${ZOO_LOG4J_PROP}\" \
+    -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryError='kill -9 %p' \
+    -cp \"$CLASSPATH\" $JVMFLAGS $ZOOMAIN \"$ZOOCFG\" > \"$_ZOO_DAEMON_OUT\" 2>&1 < /dev/null"
     ;;
 stop)
     echo -n "Stopping zookeeper ... "

src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml

@@ -533,6 +533,18 @@ server.3=zoo3:2888:3888</programlisting>
       examples) managing your ZooKeeper server ensures that if the
       process does exit abnormally it will automatically be restarted
       and will quickly rejoin the cluster.</para>
+
+      <para>It is also recommended to configure the ZooKeeper server process to
+      terminate and dump its heap if an
+      <computeroutput>OutOfMemoryError</computeroutput> occurs.  This is achieved
+      by launching the JVM with the following arguments on Linux and Windows
+      respectively.  The <filename>zkServer.sh</filename> and
+      <filename>zkServer.cmd</filename> scripts that ship with ZooKeeper set
+      these options.
+      </para>
+
+      <programlisting>-XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryError='kill -9 %p'</programlisting>
+      <programlisting>"-XX:+HeapDumpOnOutOfMemoryError" "-XX:OnOutOfMemoryError=cmd /c taskkill /pid %%%%p /t /f"</programlisting>
     </section>
 
     <section id="sc_monitoring">