HADOOP-7154. Set MALLOC_ARENA_MAX in hadoop-env.sh. Contributed by Todd Lipcon.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.0@1364929 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -6,6 +6,9 @@ Release 1.0.4 - Unreleased
 
   IMPROVEMENTS
 
+    HADOOP-7154. Should set MALLOC_ARENA_MAX in hadoop-env.sh
+    (todd via mattf)
+ 
   BUG FIXES
 
     HDFS-3652. FSEditLog failure removes the wrong edit stream when storage

bin/hadoop-config.sh

@@ -75,6 +75,11 @@ if [ "$HADOOP_HOME_WARN_SUPPRESS" = "" ] && [ "$HADOOP_HOME" != "" ]; then
   echo 1>&2
 fi
 
+# Newer versions of glibc use an arena memory allocator that causes virtual
+# memory usage to explode. This interacts badly with the many threads that
+# we use in Hadoop. Tune the variable down to prevent vmem explosion.
+export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
+
 export HADOOP_HOME=${HADOOP_PREFIX}
 export HADOOP_HOME_WARN_SUPPRESS=1