Merge of -r 633170:633171 from trunk to branch 0.16 to fix HADOOP-2809.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/branches/branch-0.16@633173 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -94,6 +94,9 @@ Release 0.16.1 - Unreleased
     HADOOP-2847.  Ensure idle cluster cleanup works even if the JobTracker
     becomes unresponsive to RPC calls. (Hemanth Yamijala via nigel)
 
+    HADOOP-2809.  Fix HOD syslog config syslog-address so that it works.
+    (Hemanth Yamijala via nigel)
+
 Release 0.16.0 - 2008-02-07
 
   INCOMPATIBLE CHANGES

src/contrib/hod/bin/hod

@@ -182,7 +182,7 @@ defList = { 'hod' : (
               False, os.path.join(rootDirectory, 'logs'), False, False),
 
              ('syslog-address', 'address', 'Syslog address.',
-              False, None, False, False),
+              False, None, False, True),
 
              ('xrs-port-range', 'range', 'XML-RPC port range n-m.',
               False, None, True, True),
@@ -292,7 +292,7 @@ defList = { 'hod' : (
               False, None, False, False),
               
              ('syslog-address', 'address', 'Syslog address.',
-              False, None, False, False),
+              False, None, False, True),
           
              ('java-home', 'directory', 'Java home directory.',
               False, None, True, False),

src/contrib/hod/bin/hodcleanup

@@ -50,6 +50,7 @@ from hodlib.Common.threads import simpleCommand
 from hodlib.Common.util import local_fqdn, tar, filter_warnings,\
                             get_exception_string, get_exception_error_string
 from hodlib.Common.logger import hodLog
+from hodlib.Common.logger import getLogger
 
 filter_warnings()
 
@@ -62,51 +63,6 @@ if os.path.exists("./VERSION"):
   VERSION = vFile.readline()
   vFile.close()
 
-def getLogger(hodRingOptions):
-  # print >>sys.stderr,"now"
-  if hodRingOptions['hodring-debug'] > 0:
-    _baseLogger = hodLog('hodring')
-    log = _baseLogger.add_logger('main')
-
-    if hodRingOptions.has_key('hodring-stream'):
-      if hodRingOptions['hodring-stream']:
-        _baseLogger.add_stream(level=hodRingOptions['hodring-debug'],
-                             addToLoggerNames=('main',))
-
-    _serviceID = hodRingOptions['service-id']
-    if hodRingOptions['hodring-log-dir']:
-      if _serviceID:
-          __logDir = os.path.join(hodRingOptions['hodring-log-dir'], "%s.%s" % (
-                                   hodRingOptions['user-id'], _serviceID))
-      else:
-          __logDir = os.path.join(hodRingOptions['hodring-log-dir'],
-                                       hodRingOptions['user-id'])
-      if not os.path.exists(__logDir):
-        os.mkdir(__logDir)
-
-      _baseLogger.add_file(logDirectory=__logDir,
-        level=hodRingOptions['hodring-debug'], addToLoggerNames=('main',))
-
-    if hodRingOptions['hodring-syslog-address']:
-      _baseLogger.add_syslog(hodRingOptions['hodring-syslog-address'],
-        level=hodRingOptions['hodring-debug'], addToLoggerNames=('main',))
-    return log
-
-def ensureLogDir(logDir):
-  """Verify that the passed in log directory exists, and if it doesn't
-  create it."""
-  if not os.path.exists(logDir):
-    try:
-      old_mask = os.umask(0)
-      os.makedirs(logDir, 01777)
-      os.umask(old_mask)
-    except Exception, e:
-      print >>sys.stderr, "Could not create log directories %s. Exception: \
-               %s. Stack Trace: %s" % (logDir, get_exception_error_string()
-               , get_exception_string())
-      raise e
-
-
 def __archive_logs(conf, log):
   # need log-destination-uri, __hadoopLogDirs, temp-dir
   status = True
@@ -206,19 +162,28 @@ def unpack():
                                                         action="store_true")
   (options, args) = parser.parse_args()
   _options= {}
+  _options['hodring'] = {}
   for opt in dir(options):
     if "--"+opt in option_list:
       _options[opt] = getattr(options,opt)
   if _options.has_key('hadoop-log-dirs') and _options['hadoop-log-dirs']:
     _options['hadoop-log-dirs'] = _options['hadoop-log-dirs'].split(",")
-  _options['hodring-debug'] = int(_options['hodring-debug'])
+  if _options.has_key('hodring-syslog-address') and _options['hodring-syslog-address']:
+    _options['hodring']['syslog-address'] = \
+        _options['hodring-syslog-address'].split(':')
+  _options['hodring']['debug']        = int(_options['hodring-debug'])
+  _options['hodring']['log-dir']      = _options['hodring-log-dir']
+  _options['hodring']['stream']      = _options['hodring-stream']
+  _options['hodring']['userid']      = _options['user-id']
+  os.putenv('PBS_JOBID', _options['service-id'] )
   return _options
  
-
 if __name__ == '__main__':  
+  log = None
   try:
     conf = unpack()
-    log = getLogger(conf,)
+    # Use the same log as hodring
+    log = getLogger(conf['hodring'],'hodring')
     log.debug("Logger initialised successfully")
     status =  __archive_logs(conf,log)
     log.info("Archive status : %s" % status)
@@ -232,4 +197,4 @@ if __name__ == '__main__':
     log.info("Cleanup successfully completed")
   except Exception, e:
     if log:
-      log.info("Stack trace:\n%s" %(get_exception_error_string(),get_exception_string()))
+      log.info("Stack trace:\n%s\n%s" %(get_exception_error_string(),get_exception_string()))

src/contrib/hod/bin/hodring

@@ -47,7 +47,7 @@ sys.path.append(libDirectory)
 from hodlib.HodRing.hodRing import HodRing
 from hodlib.Common.setup import *
 from hodlib.Common.util import filter_warnings,get_exception_string, get_exception_error_string
-from hodlib.Common.logger import hodLog
+from hodlib.Common.logger import getLogger, ensureLogDir
 
 filter_warnings()
 
@@ -133,48 +133,6 @@ defList = { 'hodring' : (
               False, 2, False, True)) 
             }            
 
-def getLogger(hodRingOptions):
-  if hodRingOptions['hodring']['debug'] > 0:
-    _baseLogger = hodLog('hodring')
-    log = _baseLogger.add_logger('main')
-
-    if hodRingOptions['hodring'].has_key('stream'):
-      if hodRingOptions['hodring']['stream']:
-        _baseLogger.add_stream(level=hodRingOptions['hodring']['debug'],
-                             addToLoggerNames=('main',))
-
-    _serviceID = os.getenv('PBS_JOBID')
-
-    if hodRingOptions['hodring'].has_key('log-dir'):
-      if _serviceID:
-          __logDir = os.path.join(hodRingOptions['hodring']['log-dir'], "%s.%s" % (
-                                   hodRingOptions['hodring']['userid'], _serviceID))
-      else:
-          __logDir = os.path.join(hodRingOptions['hodring']['log-dir'],
-                                       hodRingOptions['hodring']['userid'])
-      if not os.path.exists(__logDir):
-        os.mkdir(__logDir)
-
-      _baseLogger.add_file(logDirectory=__logDir,
-        level=hodRingOptions['hodring']['debug'], addToLoggerNames=('main',))
-
-    if hodRingOptions['hodring'].has_key('syslog-address'):
-      _baseLogger.add_syslog(hodRingOptions['hodring']['syslog-address'],
-        level=hodRingOptions['hodring']['debug'], addToLoggerNames=('main',))
-    return log
-
-def ensureLogDir(logDir):
-  """Verify that the passed in log directory exists, and if it doesn't
-  create it."""
-  if not os.path.exists(logDir):
-    try:
-      old_mask = os.umask(0)
-      os.makedirs(logDir, 01777)
-      os.umask(old_mask)
-    except Exception, e:
-      print >>sys.stderr, "Could not create log directories %s. Exception: %s. Stack Trace: %s" % (logDir, get_exception_error_string(), get_exception_string())
-      raise e
-
 if __name__ == '__main__':
 
   confDef = definition()
@@ -194,7 +152,7 @@ if __name__ == '__main__':
     if service.log:
       log = service.log
     else: 
-      log = getLogger(hodRingOptions)
+      log = getLogger(hodRingOptions['hodring'],'hodring')
 
     list = []
     
@@ -244,8 +202,9 @@ if __name__ == '__main__':
                   + ",".join(list)
 
     if hodRingOptions['hodring'].has_key('syslog-address'):
-      cmdString = cmdString + " --hodring-syslog-address " \
-                + hodRingOptions['hodring']['syslog-address']
+      syslogAddr = hodRingOptions['hodring']['syslog-address'][0] + \
+                   ':' + str(hodRingOptions['hodring']['syslog-address'][1])
+      cmdString = cmdString + " --hodring-syslog-address " + syslogAddr
     if service._cfg.has_key('pkgs'):
       cmdString = cmdString + " --pkgs " + service._cfg['pkgs']
 
@@ -292,5 +251,5 @@ if __name__ == '__main__':
       if service.log:
         log = service.log
     else:
-      log = getLogger(hodRingOptions)
+      log = getLogger(hodRingOptions['hodring'], 'hodring')
     log.error("Error in bin/hodring %s. \nStack trace:\n%s" %(get_exception_error_string(),get_exception_string()))

src/contrib/hod/bin/ringmaster

@@ -47,7 +47,7 @@ from hodlib.RingMaster.ringMaster import main
 from hodlib.Common.setup import *
 from hodlib.Common.descGenerator import *
 from hodlib.Common.util import local_fqdn, filter_warnings
-from hodlib.Common.logger import hodLog
+from hodlib.Common.logger import getLogger, ensureLogDir
 from hodlib.Common.util import get_exception_string, get_exception_error_string
 
 filter_warnings()
@@ -269,59 +269,21 @@ defList = { 'ringmaster' : (
 defOrder = [ 'ringmaster', 'hodring', 'resource_manager', 
              'gridservice-mapred', 'gridservice-hdfs' ]
 
-def getLogger(cfg):
-  user = cfg['userid']
-  baseLogger = hodLog('ringmaster')
-  log = baseLogger.add_logger('main')
-  
-  if cfg['stream']:
-    baseLogger.add_stream(level=cfg['debug'], addToLoggerNames=('main',))
-
-  if cfg.has_key('log-dir'):
-    serviceId = os.getenv('PBS_JOBID')
-    if serviceId:
-      logDir = os.path.join(cfg['log-dir'], "%s.%s" % (user, serviceId))
-    else:
-      logDir = os.path.join(cfg['log-dir'], user) 
-    if not os.path.exists(logDir):
-      os.mkdir(logDir)
-
-    baseLogger.add_file(logDirectory=logDir, level=cfg['debug'], 
-             addToLoggerNames=('main',))
-
-  if cfg.has_key('syslog-address'):
-    baseLogger.add_syslog(cfg['syslog-address'], 
-      level=cfg['debug'], addToLoggerNames=('main',))
-    
-  return log
-
-def ensureLogDir(logDir):
-  """Verify that the passed in log directory exists, and if it doesn't
-  create it."""
-  if not os.path.exists(logDir):
-    try:
-      old_mask = os.umask(0)
-      os.makedirs(logDir, 01777)
-      os.umask(old_mask)
-    except Exception, e:
-      print >>sys.stderr, "Could not create log directories %s. Exception: %s. Stack Trace: %s" % (logDir, get_exception_error_string(), get_exception_string())
-      raise e
-      
-
 if __name__ == '__main__':
   confDef = definition()
   confDef.add_defs(defList, defOrder)
   ringMasterOptions = options(confDef, "./%s [OPTIONS]" % myName, VERSION)
-  ensureLogDir(ringMasterOptions['ringmaster']['log-dir'])
   log = None
 
   try:
-    log = getLogger(ringMasterOptions['ringmaster'])
     (status, statusMsgs) = ringMasterOptions.verify()
     if not status:
       raise Exception("%s" % statusMsgs)
     ringMasterOptions.replace_escape_seqs()
     ringMasterOptions['ringmaster']['base-dir'] = rootDirectory 
+
+    ensureLogDir(ringMasterOptions['ringmaster']['log-dir'])
+    log = getLogger(ringMasterOptions['ringmaster'],'ringmaster')
     ret = main(ringMasterOptions,log)
     sys.exit(ret)
   except Exception, e:

src/contrib/hod/hodlib/Common/logger.py

@@ -376,7 +376,7 @@ class hodLog:
         def add_syslog_handler(loggerName):
             if not self.__logObjs['syslog'].has_key(loggerName):
                 loggingLevel = self.__get_logging_level(level,
-                    defaultStreamLevel)
+                    defaultSyslogLevel)
 
                 address[1] = int(address[1])
                 syslogHandler = logging.handlers.SysLogHandler(tuple(address),
@@ -738,3 +738,45 @@ class hodDummyLogger:
 
         pass
 
+def ensureLogDir(logDir):
+  """Verify that the passed in log directory exists, and if it doesn't
+  create it."""
+  if not os.path.exists(logDir):
+    try:
+      old_mask = os.umask(0)
+      os.makedirs(logDir, 01777)
+      os.umask(old_mask)
+    except Exception, e:
+      print >>sys.stderr, "Could not create log directories %s. Exception: %s. Stack Trace: %s" % (logDir, get_exception_error_string(), get_exception_string())
+      raise e
+
+def getLogger(cfg, logName):
+  if cfg['debug'] > 0:
+    user = cfg['userid']
+    baseLogger = hodLog(logName)
+    log = baseLogger.add_logger('main')
+
+    if cfg.has_key('log-dir'):
+      serviceId = os.getenv('PBS_JOBID')
+      if serviceId:
+        logDir = os.path.join(cfg['log-dir'], "%s.%s" % (user, serviceId))
+      else:
+        logDir = os.path.join(cfg['log-dir'], user) 
+      if not os.path.exists(logDir):
+        os.mkdir(logDir)
+
+      baseLogger.add_file(logDirectory=logDir, level=cfg['debug'], 
+               addToLoggerNames=('main',))
+
+    try:
+      if cfg.has_key('stream') and cfg['stream']:
+        baseLogger.add_stream(level=cfg['debug'], addToLoggerNames=('main',))
+
+      if cfg.has_key('syslog-address'):
+        baseLogger.add_syslog(cfg['syslog-address'], 
+          level=cfg['debug'], addToLoggerNames=('main',))
+    except Exception,e:
+      # Caught an exception while initialising logger
+      log.critical("%s Logger failed to initialise. Reason : %s" % (logName, e))
+      pass
+    return log

src/contrib/hod/hodlib/Common/types.py

@@ -477,11 +477,14 @@ class typeValidator:
     def __verify_address(self, type, value):
         valueInfo = self.__get_value_info()
 
-        socket = tcpSocket(value)
-        if socket.verify():
-            self.__set_value_info(valueInfo, normalized=self.normalize(type, 
+        try:
+            socket = tcpSocket(value)
+            if socket.verify():
+                self.__set_value_info(valueInfo, normalized=self.normalize(type, 
                                                                        value))
-        else:
+            else:
+                self.__set_value_info(valueInfo)
+        except:
             self.__set_value_info(valueInfo)
 
         return valueInfo