Merge of -r 633914:633915 from trunk to branch 0.16 to fix HADOOP-2925.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/branches/branch-0.16@633918 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -113,6 +113,10 @@ Release 0.16.1 - Unreleased
     datnodes flushes the block file buffered output stream before
     sending a positive ack for the packet back to the client. (dhruba)
 
+    HADOOP-2925. Fix HOD to create the mapred system directory using a 
+    naming convention that will avoid clashes in multi-user shared 
+    cluster scenario. (Hemanth Yamijala via nigel)
+
 Release 0.16.0 - 2008-02-07
 
   INCOMPATIBLE CHANGES

src/contrib/hod/bin/hod

@@ -346,7 +346,10 @@ defList = { 'hod' : (
               False, 2, False, True),
              
              ('cmd-retry-interval', 'pos_float','interval to spread retries for getting commands',
-              False, 2, False, True)) 
+              False, 2, False, True),
+
+             ('mapred-system-dir-root', 'string', 'Root under which mapreduce system directory names are generated by HOD.',
+              False, '/mapredsystem', False, False))
               }   
 
 defOrder = [ 'hod', 'ringmaster', 'hodring', 'resource_manager', 

src/contrib/hod/bin/hodcleanup

@@ -123,7 +123,7 @@ def __copy_archive_to_dfs(conf, archiveFile):
   # END LAME HACK
   
   (head, tail) = os.path.split(archiveFile)
-  destFile = os.path.join(hdfsURIMatch.group(2), conf['user-id'], conf['service-id'], tail)
+  destFile = os.path.join(hdfsURIMatch.group(2), conf['user-id'], 'hod-logs', conf['service-id'], tail)
   
   log.info("copying archive %s to DFS %s ..." % (archiveFile, destFile))
   

src/contrib/hod/bin/hodring

@@ -130,7 +130,10 @@ defList = { 'hodring' : (
               False, 2, False, True),
              
              ('cmd-retry-interval', 'pos_float','interval to spread retries for getting commands',
-              False, 2, False, True)) 
+              False, 2, False, True), 
+
+             ('mapred-system-dir-root', 'string', 'Root under which mapreduce system directory names are generated by HOD.',
+              False, '/mapredsystem', False, False))
             }            
 
 if __name__ == '__main__':

src/contrib/hod/bin/ringmaster

@@ -261,8 +261,10 @@ defList = { 'ringmaster' : (
               False, 2, False, True),
              
              ('cmd-retry-interval', 'pos_float','interval to spread retries for getting commands',
-              False, 2, False, True)) 
-
+              False, 2, False, True),
+ 
+             ('mapred-system-dir-root', 'string', 'Root under which mapreduce system directory names are generated by HOD.',
+              False, '/mapredsystem', False, False))
               }   
 
              

src/contrib/hod/hodlib/Common/util.py

@@ -250,6 +250,9 @@ def parseEquals(list):
     dict[splits[0]] = splits[1]
   return dict
 
+def getMapredSystemDirectory(mrSysDirRoot, userid, jobid):
+  return os.path.join(mrSysDirRoot, userid, 'mapredsystem', jobid)
+
 class HodInterrupt:
   def __init__(self):
     self.HodInterruptFlag = False

src/contrib/hod/hodlib/Hod/hadoop.py

@@ -57,7 +57,7 @@ class hadoopConfig:
     
     return prop
 
-  def gen_site_conf(self, confDir, tempDir, numNodes, hdfsAddr,\
+  def gen_site_conf(self, confDir, tempDir, numNodes, hdfsAddr, mrSysDir,\
              mapredAddr=None, clientParams=None, serverParams=None,\
              finalServerParams=None, clusterFactor=None):
     if not mapredAddr:
@@ -76,9 +76,7 @@ class hadoopConfig:
                     'hadoop.tmp.dir'        : confDir, \
                     'dfs.client.buffer.dir' : tempDir, }
 
-    mapredAddrSplit = mapredAddr.split(":")
-    mapredsystem = os.path.join('/mapredsystem', mapredAddrSplit[0])
-    paramsDict['mapred.system.dir'] = mapredsystem 
+    paramsDict['mapred.system.dir'] = mrSysDir
     
     # mapred-default.xml is no longer used now.
     numred = int(math.floor(clusterFactor * (int(numNodes) - 1)))
@@ -515,8 +513,10 @@ class hadoopCluster:
                     os.makedirs(tempDir)
                   tempDir = os.path.join( tempDir, self.__cfg['hod']['userid']\
                                   + "." + self.jobId )
+                  mrSysDir = getMapredSystemDirectory(self.__cfg['hodring']['mapred-system-dir-root'],\
+                                      self.__cfg['hod']['userid'], self.jobId)
                   self.__hadoopCfg.gen_site_conf(clusterDir, tempDir, min,\
-                            hdfsAddr, mapredAddr, clientParams,\
+                            hdfsAddr, mrSysDir, mapredAddr, clientParams,\
                             serverParams, finalServerParams,\
                             clusterFactor)
                   self.__log.info("hadoop-site.xml at %s" % clusterDir)

src/contrib/hod/hodlib/HodRing/hodRing.py

@@ -25,7 +25,7 @@ from xml.dom import getDOMImplementation
 from pprint import pformat
 from optparse import OptionParser
 from urlparse import urlparse
-from hodlib.Common.util import local_fqdn, parseEquals
+from hodlib.Common.util import local_fqdn, parseEquals, getMapredSystemDirectory
 from hodlib.Common.tcp import tcpSocket, tcpError 
 
 binfile = sys.path[0]
@@ -150,10 +150,12 @@ class CommandDesc:
 class HadoopCommand:
   """Runs a single hadoop command"""
     
-  def __init__(self, id, desc, tempdir, tardir, log, javahome, restart=False):
+  def __init__(self, id, desc, tempdir, tardir, log, javahome, 
+                mrSysDir, restart=False):
     self.desc = desc
     self.log = log
     self.javahome = javahome
+    self.__mrSysDir = mrSysDir
     self.program = desc.getProgram()
     self.name = desc.getName()
     self.workdirs = desc.getWorkDirs()
@@ -218,7 +220,7 @@ class HadoopCommand:
       
       self.filledInKeyVals.append(keyvalpair)
       if ( v == "fillindir"):
-        v = os.path.join('/mapredsystem', local_fqdn())
+        v = self.__mrSysDir
         pass
       
       prop = None
@@ -509,8 +511,11 @@ class HodRing(hodBaseService):
     id = 0
     for desc in self._cfg['commanddesc']:
       self.log.debug(pprint.pformat(desc.dict))
+      mrSysDir = getMapredSystemDirectory(self._cfg['mapred-system-dir-root'],
+                          self._cfg['userid'], self._cfg['service-id'])
+      self.log.debug('mrsysdir is %s' % mrSysDir)
       cmd = HadoopCommand(id, desc, self.__tempDir, self.__pkgDir, self.log, 
-                          self._cfg['java-home'], restart)
+                          self._cfg['java-home'], mrSysDir, restart)
     
       self.__hadoopLogDirs.append(cmd.logdir)
       self.log.debug("hadoop log directory: %s" % self.__hadoopLogDirs)

src/contrib/hod/support/logcondense.py

@@ -125,7 +125,7 @@ def runcondense():
   for line in stdout:
     m = re.match("^(.*?)\s.*$", line)
     filename = m.group(1)
-    # file name format: <prefix>/<user>/<jobid>/[0-1]-[jobtracker|tasktracker|datanode|namenode|]-hostname-YYYYMMDDtime-random.tar.gz
+    # file name format: <prefix>/<user>/hod-logs/<jobid>/[0-1]-[jobtracker|tasktracker|datanode|namenode|]-hostname-YYYYMMDDtime-random.tar.gz
     # first strip prefix:
     if filename.startswith(options.log):
       filename = filename.lstrip(options.log)
@@ -135,7 +135,7 @@ def runcondense():
       continue
     
     # Now get other details from filename.
-    k = re.match("/(.*)/(.*)/.*-.*-([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9]).*$", filename)
+    k = re.match("/(.*)/.*/(.*)/.*-.*-([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9]).*$", filename)
     if k:
       username = k.group(1)
       jobid =  k.group(2)
@@ -159,7 +159,7 @@ def runcondense():
       if (diff.days > options.days):
         desttodel = filename
         if not toPurge.has_key(jobid):
-          toPurge[jobid] = options.log.rstrip("/") + "/" + username + "/" + jobid
+          toPurge[jobid] = options.log.rstrip("/") + "/" + username + "/hod-logs/" + jobid
 
   for job in toPurge.keys():
     for prefix in deletedNamePrefixes:

src/docs/src/documentation/content/xdocs/hod_config_guide.xml

@@ -85,6 +85,10 @@
                              be picked for use to run an HTTP server.</li>
           
           <li>java-home: Location of Java to be used by Hadoop.</li>
+          <li>syslog-address: Address to which a syslog daemon is bound to. The format 
+                              of the value is host:port. If configured, HOD log messages
+                              will be logged to syslog using this value.</li>
+                              
         </ul>
       </section>
       
@@ -202,6 +206,47 @@
           <li>final-server-params: Same as above, except they will be marked final.</li>
         </ul>
       </section>
+
+      <section>
+        <title>3.7 hodring options</title>
+
+        <ul>
+          <li>mapred-system-dir-root: Directory in the DFS under which HOD will
+                                      generate sub-directory names and pass the full path
+                                      as the value of the 'mapred.system.dir' configuration 
+                                      parameter to Hadoop daemons. The format of the full 
+                                      path will be value-of-this-option/userid/mapredsystem/cluster-id.
+                                      Note that the directory specified here should be such
+                                      that all users can create directories under this, if
+                                      permissions are enabled in HDFS. Setting the value of
+                                      this option to /user will make HOD use the user's
+                                      home directory to generate the mapred.system.dir value.</li>
+
+          <li>log-destination-uri: URL describing a path in an external, static DFS or the 
+                                   cluster node's local file system where HOD will upload 
+                                   Hadoop logs when a cluster is deallocated. To specify a 
+                                   DFS path, use the format 'hdfs://path'. To specify a 
+                                   cluster node's local file path, use the format 'file://path'.
+
+                                   When clusters are deallocated by HOD, the hadoop logs will
+                                   be deleted as part of HOD's cleanup process. In order to
+                                   persist these logs, you can use this configuration option.
+
+                                   The format of the path is 
+                                   value-of-this-option/userid/hod-logs/cluster-id
+
+                                   Note that the directory you specify here must be such that all
+                                   users can create sub-directories under this. Setting this value
+                                   to hdfs://user will make the logs come in the user's home directory
+                                   in DFS.</li>
+
+          <li>pkgs: Installation directory, under which bin/hadoop executable is located. This will
+                    be used by HOD to upload logs if a HDFS URL is specified in log-destination-uri
+                    option. Note that this is useful if the users are using a tarball whose version
+                    may differ from the external, static HDFS version.</li>
+                                      
+        </ul>
+      </section>
     </section>
   </body>
 </document>