hadoop/src/contrib/hod/hodlib/Common/setup.py

#Licensed to the Apache Software Foundation (ASF) under one
#or more contributor license agreements.  See the NOTICE file
#distributed with this work for additional information
#regarding copyright ownership.  The ASF licenses this file
#to you under the Apache License, Version 2.0 (the
#"License"); you may not use this file except in compliance
#with the License.  You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

#Unless required by applicable law or agreed to in writing, software
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.
# $Id:setup.py 5158 2007-04-09 00:14:35Z zim $
# $Id:setup.py 5158 2007-04-09 00:14:35Z zim $
#
#------------------------------------------------------------------------------

"""'setup' provides for reading and verifing configuration files based on
   Python's SafeConfigParser class."""

import sys, os, re, pprint

from ConfigParser import SafeConfigParser
from optparse import OptionParser, IndentedHelpFormatter, OptionGroup
from util import get_perms, replace_escapes
from types import typeValidator, typeValidatorInstance, is_valid_type, \
                  typeToString
from hodlib.Hod.hod import hodHelp

reEmailAddress = re.compile("^.*@.*$")
reEmailDelimit = re.compile("@")
reComma = re.compile("\s*,\s*")
reDot = re.compile("\.")
reCommentHack = re.compile("^.*?\s+#|;.*", flags=re.S)
reCommentNewline = re.compile("\n|\r$")
reKeyVal = r"(?<!\\)="
reKeyVal = re.compile(reKeyVal)
reKeyValList = r"(?<!\\),"
reKeyValList = re.compile(reKeyValList)

errorPrefix = 'error'
requiredPerms = '0660'

class definition:
    def __init__(self):
        """Generates a configuration definition object."""
        self.__def = {}
        self.__defOrder = []

    def __repr__(self):
        return pprint.pformat(self.__def)  

    def __getitem__(self, section):
        return self.__def[section]

    def __iter__(self):
        return iter(self.__def)

    def sections(self):
        """Returns a list of sections/groups."""
        
        if len(self.__defOrder):
            return self.__defOrder
        else:  
            return self.__def.keys()
      
    def add_section(self, section):
        """Add a configuration section / option group."""
        
        if self.__def.has_key(section):
            raise Exception("Section already exists: '%s'" % section)
        else:
            self.__def[section] = {}

    def add_def(self, section, var, type, desc, help = True, default = None, 
                req = True, validate = True, short = None):
        """ Add a variable definition.
        
            section  - section name
            var      - variable name
            type     - valid hodlib.types
            desc     - description of variable
            help     - display help for this variable
            default  - default value
            req      - bool, requried?
            validate - bool, validate type value?
            short    - short symbol (1 character),
            help     - bool, display help?"""
            
        if self.__def.has_key(section):
            if not is_valid_type(type):
                raise Exception("Type (type) is invalid: %s.%s - '%s'" % (section, var, 
                                                                type))
            if not isinstance(desc, str):
                raise Exception("Description (desc) must be a string: %s.%s - '%s'" % (
                    section, var, desc))
            if not isinstance(req, bool):
                raise Exception("Required (req) must be a bool: %s.%s - '%s'" % (section, 
                                                                       var, 
                                                                       req))
            if not isinstance(validate, bool):
                raise Exception("Validate (validate) must be a bool: %s.%s - '%s'" % (
                    section, var, validate))
              
            if self.__def[section].has_key(var):
                raise Exception("Variable name already defined: '%s'" % var)
            else:
                self.__def[section][var] = { 'type'     : type,
                                             'desc'     : desc,
                                             'help'     : help,
                                             'default'  : default,
                                             'req'      : req,
                                             'validate' : validate,
                                             'short'    : short }                
        else:    
            raise Exception("Section does not exist: '%s'" % section)
          
    def add_defs(self, defList, defOrder=None):
        """ Add a series of definitions.
        
            defList = { section0 : ((name0, 
                                     type0, 
                                     desc0, 
                                     help0,
                                     default0,
                                     req0, 
                                     validate0,
                                     short0),
                                  ....
                                    (nameN, 
                                     typeN, 
                                     descN,
                                     helpN, 
                                     defaultN, 
                                     reqN, 
                                     validateN,
                                     shortN)),             
                           ....
                           
                        sectionN : ... }
                        
            Where the short synmbol is optional and can only be one char."""
                        
        for section in defList.keys():
            self.add_section(section)
            for defTuple in defList[section]:
                if isinstance(defTuple, tuple): 
                    if len(defTuple) < 7:
                        raise Exception(
                            "section %s is missing an element: %s" % (
                            section, pprint.pformat(defTuple)))
                else:
                    raise Exception("section %s of defList is not a tuple" % 
                                    section)
                
                if len(defTuple) == 7:
                    self.add_def(section, defTuple[0], defTuple[1], 
                                 defTuple[2], defTuple[3], defTuple[4], 
                                 defTuple[5], defTuple[6])
                else:
                    self.add_def(section, defTuple[0], defTuple[1], 
                                 defTuple[2], defTuple[3], defTuple[4], 
                                 defTuple[5], defTuple[6], defTuple[7])                     
        if defOrder:
            for section in defOrder:
                if section in self.__def:
                    self.__defOrder.append(section)
                    
            for section in self.__def:
                if not section in defOrder:
                    raise Exception(
                        "section %s is missing from specified defOrder." % 
                        section)
            
class baseConfig:
    def __init__(self, configDef, originalDir=None):
        self.__toString = typeToString()
        self.__validated = False
        self._configDef = configDef
        self._options = None
        self._mySections = []
        self._dict = {}
        self.configFile = None
        self.__originalDir = originalDir

        if self._configDef:
            self._mySections = configDef.sections()

    def __repr__(self):
        """Returns a string representation of a config object including all
           normalizations."""

        print_string = '';
        for section in self._mySections:
            print_string = "%s[%s]\n" % (print_string, section)
            options = self._dict[section].keys()
            for option in options:
                print_string = "%s%s = %s\n" % (print_string, option,
                    self._dict[section][option])

            print_string = "%s\n" % (print_string)

        print_string = re.sub("\n\n$", "", print_string)

        return print_string

    def __getitem__(self, section):
        """ Returns a dictionary of configuration name and values by section.
        """
        return self._dict[section]

    def __setitem__(self, section, value):
        self._dict[section] = value

    def __iter__(self):
        return iter(self._dict)

    def has_key(self, section):
        status = False
        if section in self._dict:
            status = True
            
        return status

    # Prints configuration error messages
    def var_error(self, section, option, *addData):
        errorStrings = []  
        if not self._dict[section].has_key(option):
          self._dict[section][option] = None
        errorStrings.append("%s: invalid '%s' specified in section %s (--%s.%s): %s" % (
            errorPrefix, option, section, section, option, self._dict[section][option]))

        if addData:
            errorStrings.append("%s: additional info: %s\n" % (errorPrefix,
                addData[0]))
        return errorStrings

    def var_error_suggest(self, errorStrings):
        if self.configFile:
            errorStrings.append("Check your command line options and/or " + \
                              "your configuration file %s" % self.configFile)
    
    def __get_args(self, section):
        def __dummyToString(type, value):
            return value
        
        toString = __dummyToString
        if self.__validated:
            toString = self.__toString
            
        args = []
        if isinstance(self._dict[section], dict):
            for option in self._dict[section]:
                if section in self._configDef and \
                option in self._configDef[section]:
                  if self._configDef[section][option]['type'] == 'bool':
                    if self._dict[section][option] == 'True' or \
                        self._dict[section][option] == True:
                        args.append("--%s.%s" % (section, option))
                  else:
                    args.append("--%s.%s" % (section, option))
                    args.append(toString(
                           self._configDef[section][option]['type'], 
                           self._dict[section][option]))
        else:
            if section in self._configDef:
              if self._configDef[section][option]['type'] == 'bool':
                if self._dict[section] == 'True' or \
                    self._dict[section] == True:
                    args.append("--%s" % section)
              else:
                if self._dict[section] != 'config':
                  args.append("--%s" % section)
                  args.append(toString(self._configDef[section]['type'], 
                                             self._dict[section]))
                    
        return args
                
    def values(self):
        return self._dict.values()
      
    def keys(self):
        return self._dict.keys()
    
    def get_args(self, exclude=None, section=None):
        """Retrieve a tuple of config arguments."""
        
        args = []
        if section:
            args = self.__get_args(section)
        else:
            for section in self._dict:
                if exclude:
                    if not section in exclude:
                        args.extend(self.__get_args(section))
                else:
                    args.extend(self.__get_args(section))
        
        return tuple(args)
        
    def verify(self):
        """Verifies each configuration variable, using the configValidator
           class, based on its type as defined by the dictionary configDef.
           Upon encountering a problem an error is printed to STDERR and
           false is returned."""
        
        oldDir = os.getcwd()
        if self.__originalDir:
          os.chdir(self.__originalDir)
        
        status = True
        statusMsgs = []
	
        if self._configDef:
            errorCount = 0
            configValidator = typeValidator(self.__originalDir)

            # foreach section and option by type string as defined in configDef
            #   add value to be validated to validator
            for section in self._mySections:
                for option in self._configDef[section].keys():
                    configVarName = "%s.%s" % (section, option)

                    if self._dict[section].has_key(option):
                        if self._configDef[section][option].has_key('validate'):
                            if self._configDef[section][option]['validate']:
                                # is the section.option needed to be validated?
                                configValidator.add(configVarName,
                                    self._configDef[section][option]['type'],
                                    self._dict[section][option])
                            else:
                                # If asked not to validate, just normalize
                                self[section][option] = \
                                    configValidator.normalize(
                                    self._configDef[section][option]['type'], 
                                    self._dict[section][option])
                            if self._configDef[section][option]['default'] != \
                                None:
                                self._configDef[section][option]['default'] = \
                                    configValidator.normalize(
                                    self._configDef[section][option]['type'],
                                    self._configDef[section][option]['default']
                                    )
                                self._configDef[section][option]['default'] = \
                                    self.__toString(
                                    self._configDef[section][option]['type'], 
                                    self._configDef[section][option]['default']
                                    )
                        else:        
                            # This should not happen. Just in case, take this as 'to be validated' case.
                            configValidator.add(configVarName,
                                self._configDef[section][option]['type'],
                                self._dict[section][option])
                    elif self._configDef[section][option]['req']:
                        statusMsgs.append("%s: %s.%s is not defined."
                             % (errorPrefix, section, option))
                        errorCount = errorCount + 1                         

            configValidator.validate()

            for valueInfo in configValidator.validatedInfo:
                sectionsOptions = reDot.split(valueInfo['name'])

                if valueInfo['isValid'] == 1:
                    self._dict[sectionsOptions[0]][sectionsOptions[1]] = \
                        valueInfo['normalized']
                else:
                    if valueInfo['errorData']:
                        statusMsgs.extend(self.var_error(sectionsOptions[0],
                            sectionsOptions[1], valueInfo['errorData']))
                    else:
                        statusMsgs.extend(self.var_error(sectionsOptions[0],
                            sectionsOptions[1]))
                    errorCount = errorCount + 1

            if errorCount > 1:
                statusMsgs.append( "%s: %s problems found." % (
                    errorPrefix, errorCount))
                self.var_error_suggest(statusMsgs)
                status = False
            elif errorCount > 0:
                statusMsgs.append( "%s: %s problem found." % (
                    errorPrefix, errorCount))
                self.var_error_suggest(statusMsgs)
                status = False
        
        self.__validated = True

        if self.__originalDir:
          os.chdir(oldDir)

        return status,statusMsgs

    def normalizeValue(self, section, option)  :
      return typeValidatorInstance.normalize(
                                  self._configDef[section][option]['type'],
                                  self[section][option])

    def validateValue(self, section, option):
      # Validates a section.option and exits on error
      valueInfo = typeValidatorInstance.verify(
                                  self._configDef[section][option]['type'],
                                  self[section][option])
      if valueInfo['isValid'] == 1:
        return []
      else:
        if valueInfo['errorData']:
          return self.var_error(section, option, valueInfo['errorData'])
        else:
          return self.var_error(section, option)

class config(SafeConfigParser, baseConfig):
    def __init__(self, configFile, configDef=None, originalDir=None, 
                 options=None, checkPerms=False):
        """Constructs config object.

           configFile - configuration file to read
           configDef  - definition object
           options    - options object
           checkPerms - check file permission on config file, 0660

           sample configuration file:

            [snis]
            modules_dir  = modules/       ; location of infoModules
            md5_defs_dir = etc/md5_defs   ; location of infoTree md5 defs
            info_store   = var/info       ; location of nodeInfo store
            cam_daemon   = localhost:8200 ; cam daemon address"""


        SafeConfigParser.__init__(self)
        baseConfig.__init__(self, configDef, originalDir)

        if(os.path.exists(configFile)):
          self.configFile = configFile
        else:
          raise IOError
        
        self._options = options
        
	## UNUSED CODE : checkPerms is never True
  ## zim: this code is used if one instantiates config() with checkPerms set to
  ## True.
        if checkPerms: self.__check_perms()

        self.read(configFile)

        self._configDef = configDef
        if not self._configDef:
            self._mySections = self.sections()

        self.__initialize_config_dict()

    def __initialize_config_dict(self):
        """ build a dictionary of config vars keyed by section name defined in
           configDef, if options defined override config"""

        for section in self._mySections:
            items = self.items(section)
            self._dict[section] = {}

            # First fill self._dict with whatever is given in hodrc.
            # Going by this, options given at the command line either override
            # options in hodrc, or get appended to the list, like for
            # hod.client-params. Note that after this dict has _only_ hodrc
            # params
            for keyValuePair in items:
                # stupid commenting bug in ConfigParser class, lines without an
                #  option value pair or section required that ; or # are at the
                #  beginning of the line, :(
                newValue = reCommentHack.sub("", keyValuePair[1])
                newValue = reCommentNewline.sub("", newValue)
                self._dict[section][keyValuePair[0]] = newValue
            # end of filling with options given in hodrc
            # now start filling in command line options
            if self._options:    
                for option in self._configDef[section].keys():
                    if self._options[section].has_key(option):
                        # the user has given an option
                        compoundOpt = "%s.%s" %(section,option)
                        if ( compoundOpt == \
                              'gridservice-mapred.final-server-params' \
                              or compoundOpt == \
                                    'gridservice-hdfs.final-server-params' \
                              or compoundOpt == \
                                    'gridservice-mapred.server-params' \
                              or compoundOpt == \
                                    'gridservice-hdfs.server-params' \
                              or compoundOpt == \
                                    'hod.client-params' ):
                 
                           if ( compoundOpt == \
                              'gridservice-mapred.final-server-params' \
                              or compoundOpt == \
                                    'gridservice-hdfs.final-server-params' ):
                              overwrite = False
                           else: overwrite = True

                           # Append to the current list of values in self._dict
                           if not self._dict[section].has_key(option):
                             self._dict[section][option] = ""
                           dictOpts = reKeyValList.split(self._dict[section][option])
                           dictOptsKeyVals = {}
                           for opt in dictOpts:
                              if opt != '':
                                # when dict _has_ params from hodrc
                                if reKeyVal.search(opt):
                                  (key, val) = reKeyVal.split(opt,1)
                                  # we only consider the first '=' for splitting
                                  # we do this to support passing params like
                                  # mapred.child.java.opts=-Djava.library.path=some_dir
                                  # Even in case of an invalid error like unescaped '=',
                                  # we don't want to fail here itself. We leave such errors 
                                  # to be caught during validation which happens after this
                                  dictOptsKeyVals[key] = val
                                else: 
                                  # this means an invalid option. Leaving it
                                  #for config.verify to catch
                                  dictOptsKeyVals[opt] = None
                                
                           cmdLineOpts = reKeyValList.split(self._options[section][option])

                           for opt in cmdLineOpts:
                              if reKeyVal.search(opt):
                                # Same as for hodrc options. only consider
                                # the first =
                                ( key, val ) = reKeyVal.split(opt,1)
                              else:
                                key = opt
                                val = None
                              # whatever is given at cmdline overrides
                              # what is given in hodrc only for non-final params
                              if dictOptsKeyVals.has_key(key):
                                if overwrite:
                                  dictOptsKeyVals[key] = val
                              else: dictOptsKeyVals[key] = val
                              
                           self._dict[section][option] = ""
                           for key in dictOptsKeyVals:
                              if self._dict[section][option] == "":
                                if dictOptsKeyVals[key]:
                                  self._dict[section][option] = key + "=" + \
                                    dictOptsKeyVals[key]
                                else: #invalid option. let config.verify catch
                                  self._dict[section][option] = key
                              else:
                                if dictOptsKeyVals[key]:
                                  self._dict[section][option] = \
                                    self._dict[section][option] + "," + key + \
                                      "=" + dictOptsKeyVals[key]
                                else:  #invalid option. let config.verify catch
                                  self._dict[section][option] = \
                                    self._dict[section][option] + "," + key

                        else:
                             # for rest of the options, that don't need
                            # appending business.
                            # options = cmdline opts + defaults
                            # dict    = hodrc opts only
                            # only non default opts can overwrite any opt
                            # currently in dict
                           if not self._dict[section].has_key(option):
                              # options not mentioned in hodrc
                              self._dict[section][option] = \
                                               self._options[section][option]
                           elif self._configDef[section][option]['default'] != \
                                               self._options[section][option]:
                              # option mentioned in hodrc but user has given a
                              # non-default option
                              self._dict[section][option] = \
                                               self._options[section][option]

    ## UNUSED METHOD
    ## zim: is too :)
    def __check_perms(self):
        perms = None
        if self._options:  
            try:
                perms = get_perms(self.configFile)
            except OSError, data:
                self._options.print_help()
                raise Exception("*** could not find config file: %s" % data)
                sys.exit(1)
        else:
            perms = get_perms(self.configFile)
               
        if perms != requiredPerms:
            error = "*** '%s' has invalid permission: %s should be %s\n" % \
                (self.configFile, perms, requiredPerms)
            raise Exception( error)
            sys.exit(1)

    def replace_escape_seqs(self):
      """ replace any escaped characters """
      replace_escapes(self)

class formatter(IndentedHelpFormatter):
    def format_option_strings(self, option):
        """Return a comma-separated list of option strings & metavariables."""
        if option.takes_value():
            metavar = option.metavar or option.dest.upper()
            short_opts = [sopt
                          for sopt in option._short_opts]
            long_opts = [self._long_opt_fmt % (lopt, metavar)
                         for lopt in option._long_opts]
        else:
            short_opts = option._short_opts
            long_opts = option._long_opts

        if self.short_first:
            opts = short_opts + long_opts
        else:
            opts = long_opts + short_opts

        return ", ".join(opts)    

class options(OptionParser, baseConfig):

    def __init__(self, optionDef, usage, version, originalDir=None, 
                 withConfig=False, defaultConfig=None, defaultLocation=None,
                 name=None):
        """Constructs and options object.
         
           optionDef     - definition object
           usage         - usage statement
           version       - version string
           withConfig    - used in conjunction with a configuration file
           defaultConfig - default configuration file
        
        """
        OptionParser.__init__(self, usage=usage)
        baseConfig.__init__(self, optionDef, originalDir)
        
        self.formatter = formatter(4, max_help_position=100, width=180, 
                                   short_first=1)
        
        self.__name = name
        self.__version = version
        self.__withConfig = withConfig
        self.__defaultConfig = defaultConfig
        self.__defaultLoc = defaultLocation
        self.args = []
        self.__optionList = []
        self.__compoundOpts = []
        self.__shortMap = {}
        self.__alphaString = 'abcdefghijklmnopqrstuvxyzABCDEFGHIJKLMNOPQRSTUVXYZ1234567890'
        self.__alpha = []
        self.__parsedOptions = {}
        self.__reserved = [ 'h' ]
        
        self.__orig_grps = []
        self.__orig_grp_lists = {}
        self.__orig_option_list = []
        
        self.__display_grps = []
        self.__display_grp_lists = {}
        self.__display_option_list = [] 
        
        self.config = None
        
        if self.__withConfig:
            self.__reserved.append('c')
        self.__reserved.append('v')
        
        self.__gen_alpha()            

        # build self.__optionList, so it contains all the options that are
        # possible. the list elements are of the form section.option
        for section in self._mySections:
            if self.__withConfig and section == 'config':
                raise Exception(
                    "withConfig set 'config' cannot be used as a section name")
            for option in self._configDef[section].keys():
                if '.' in option:
                    raise Exception("Options cannot contain: '.'")
                elif self.__withConfig and option == 'config':
                    raise Exception(
                        "With config set, option config is not allowed.")
                elif self.__withConfig and option == 'verbose-help':
                    raise Exception(
                        "With config set, option verbose-help is not allowed.")                 
                self.__optionList.append(self.__splice_compound(section, 
                                                                option))
        self.__build_short_map()
        self.__add_options()
        self.__init_display_options() 
        
        (self.__parsedOptions, self.args) = self.parse_args()

        # Now process the positional arguments only for the client side
        if self.__name == 'hod':

          hodhelp = hodHelp()

          _operation = getattr(self.__parsedOptions,'hod.operation')
          _script = getattr(self.__parsedOptions, 'hod.script')
          nArgs = self.args.__len__()
          if _operation:
            # -o option is given
            if nArgs != 0:
              self.error('invalid syntax : command and operation(-o) cannot coexist')
          elif nArgs == 0 and _script:
            # for a script option, without subcommand: hod -s script ...
            pass
          elif nArgs == 0:
            print "Usage: ",hodhelp.help()
            sys.exit(0)
          else:
            # subcommand is given
            cmdstr = self.args[0] # the subcommand itself
            cmdlist = hodhelp.ops
            if cmdstr not in cmdlist:
              print "Usage: ", hodhelp.help()
              sys.exit(2)

            numNodes = None
            clusterDir = None
            # Check which subcommand. cmdstr  = subcommand itself now.
            if cmdstr == "allocate":
              clusterDir = getattr(self.__parsedOptions, 'hod.clusterdir')
              numNodes = getattr(self.__parsedOptions, 'hod.nodecount')
 
              if not clusterDir or not numNodes:
                print hodhelp.usage(cmdstr)
                sys.exit(3)

              cmdstr = cmdstr + ' ' + clusterDir + ' ' + numNodes

              setattr(self.__parsedOptions,'hod.operation', cmdstr)
 
            elif cmdstr == "deallocate" or cmdstr == "info":
              clusterDir = getattr(self.__parsedOptions, 'hod.clusterdir')

              if not clusterDir:
                print hodhelp.usage(cmdstr)
                sys.exit(3)
 
              cmdstr = cmdstr + ' ' + clusterDir
              setattr(self.__parsedOptions,'hod.operation', cmdstr)

            elif cmdstr == "list":
              setattr(self.__parsedOptions,'hod.operation', cmdstr)
              pass
 
            elif cmdstr == "script":
              clusterDir = getattr(self.__parsedOptions, 'hod.clusterdir')
              numNodes = getattr(self.__parsedOptions, 'hod.nodecount')
              originalDir = getattr(self.__parsedOptions, 'hod.original-dir')

              if originalDir and clusterDir:
                self.remove_exit_code_file(originalDir, clusterDir)

              if not _script or not clusterDir or not numNodes:
                print hodhelp.usage(cmdstr)
                sys.exit(3)
              pass

            elif cmdstr == "help":
              if nArgs == 1:
                self.print_help()
                sys.exit(0)
              elif nArgs != 2:
                self.print_help()
                sys.exit(3)
              elif self.args[1] == 'options':
                self.print_options()
                sys.exit(0)
              cmdstr = cmdstr + ' ' + self.args[1]
              setattr(self.__parsedOptions,'hod.operation', cmdstr)

        # end of processing for arguments on the client side

        if self.__withConfig:
            self.config = self.__parsedOptions.config
            if not self.config:
                self.error("configuration file must be specified")
            if not os.path.isabs(self.config):
                # A relative path. Append the original directory which would be the
                # current directory at the time of launch
                try:  
                    origDir = getattr(self.__parsedOptions, 'hod.original-dir')
                    if origDir is not None:
                        self.config = os.path.join(origDir, self.config)
                        self.__parsedOptions.config = self.config
                except AttributeError, e:
                    self.error("hod.original-dir is not defined.\
                                   Cannot get current directory")
            if not os.path.exists(self.config):
                if self.__defaultLoc and not re.search("/", self.config):
                    self.__parsedOptions.config = os.path.join(
                        self.__defaultLoc, self.config)
        self.__build_dict()   

    def norm_cluster_dir(self, orig_dir, directory):
        directory = os.path.expanduser(directory)
        if not os.path.isabs(directory):
            directory = os.path.join(orig_dir, directory)
        directory = os.path.abspath(directory)

        return directory

    def remove_exit_code_file(self, orig_dir, dir):
        try:
            dir = self.norm_cluster_dir(orig_dir, dir)
            if os.path.exists(dir):
                exit_code_file = os.path.join(dir, "script.exitcode")
                if os.path.exists(exit_code_file):
                    os.remove(exit_code_file)
        except:
            print >>sys.stderr, "Could not remove the script.exitcode file."
    
    def __init_display_options(self):
        self.__orig_option_list = self.option_list[:]
        optionListTitleMap = {}
        for option in self.option_list:
            optionListTitleMap[option._long_opts[0]] = option
      
        self.__orig_grps = self.option_groups[:]
        for group in self.option_groups:
            self.__orig_grp_lists[group.title] = group.option_list[:]
                                    
        groupTitleMap = {}
        optionTitleMap = {}
        for group in self.option_groups:
            groupTitleMap[group.title] = group
            optionTitleMap[group.title] = {}
            for option in group.option_list:
                (sectionName, optionName) = \
                    self.__split_compound(option._long_opts[0])
                optionTitleMap[group.title][optionName] = option
          
        for section in self._mySections:
            for option in self._configDef[section]:
                if self._configDef[section][option]['help']:
                    if groupTitleMap.has_key(section):
                        if not self.__display_grp_lists.has_key(section):
                            self.__display_grp_lists[section] = []
                        self.__display_grp_lists[section].append(
                            optionTitleMap[section][option])
                    
                    try:    
                        self.__display_option_list.append(
                            optionListTitleMap["--" + self.__splice_compound(
                            section, option)])
                    except KeyError:
                        pass
        try:
            self.__display_option_list.append(optionListTitleMap['--config'])
        except KeyError:
            pass
          
        self.__display_option_list.append(optionListTitleMap['--help'])
        self.__display_option_list.append(optionListTitleMap['--verbose-help'])
        self.__display_option_list.append(optionListTitleMap['--version'])
                    
        self.__display_grps = self.option_groups[:]             
        for section in self._mySections:
            if self.__display_grp_lists.has_key(section):
                self.__orig_grp_lists[section] = \
                    groupTitleMap[section].option_list
            else:
                try:
                    self.__display_grps.remove(groupTitleMap[section])
                except KeyError:
                    pass
                
    def __gen_alpha(self):
        assignedOptions = []
        for section in self._configDef:
            for option in self._configDef[section]:
                if self._configDef[section][option]['short']:
                    assignedOptions.append(
                        self._configDef[section][option]['short'])
        
        for symbol in self.__alphaString:
            if not symbol in assignedOptions:
                self.__alpha.append(symbol)

    def __splice_compound(self, section, option):
        return "%s.%s" % (section, option)
        
    def __split_compound(self, compound):    
        return compound.split('.')
        
    def __build_short_map(self):
        """ build a short_map of parametername : short_option. This is done
        only for those parameters that don't have short options already
        defined in configDef.
        If possible, the first letter in the option that is not already
        used/reserved as a short option is allotted. Otherwise the first
        letter in __alpha that isn't still used is allotted.
        e.g. { 'hodring.java-home': 'T', 'resource_manager.batch-home': 'B' }
        """

        optionsKey = {}
        for compound in self.__optionList:
            (section, option) = self.__split_compound(compound)
            if not optionsKey.has_key(section):
                optionsKey[section] = []
            optionsKey[section].append(option)
        
        for section in self._configDef.sections():
            options = optionsKey[section]
            options.sort()
            for option in options:
                if not self._configDef[section][option]['short']:
                    compound = self.__splice_compound(section, option)
                    shortOptions = self.__shortMap.values()
                    for i in range(0, len(option)):
                        letter = option[i]
                        letter = letter.lower()
                        if letter in self.__alpha:
                            if not letter in shortOptions and \
                                not letter in self.__reserved:
                                self.__shortMap[compound] = letter
                                break
                    if not self.__shortMap.has_key(compound):
                        for i in range(0, len(self.__alpha)):
                            letter = self.__alpha[i]
                            if not letter in shortOptions and \
                                not letter in self.__reserved:
                                self.__shortMap[compound] = letter

    def __add_option(self, config, compoundOpt, section, option, group=None):
        addMethod = self.add_option
        if group: addMethod=group.add_option
        
        self.__compoundOpts.append(compoundOpt)
        
        if compoundOpt == 'gridservice-mapred.final-server-params' or \
           compoundOpt == 'gridservice-hdfs.final-server-params' or \
           compoundOpt == 'gridservice-mapred.server-params' or \
           compoundOpt == 'gridservice-hdfs.server-params' or \
           compoundOpt == 'hod.client-params':
          _action = 'append'
        elif config[section][option]['type'] == 'bool':
          _action = 'store_true'
        else:
          _action = 'store'

        if self.__shortMap.has_key(compoundOpt):
          addMethod("-" + self.__shortMap[compoundOpt],
                          "--" + compoundOpt, dest=compoundOpt, 
                          action= _action, 
                          metavar=config[section][option]['type'],
                          default=config[section][option]['default'],
                          help=config[section][option]['desc'])
        else:
          if config[section][option]['short']:
            addMethod("-" + config[section][option]['short'], 
                              "--" + compoundOpt, dest=compoundOpt, 
                              action= _action,
                              metavar=config[section][option]['type'],
                              default=config[section][option]['default'],
                              help=config[section][option]['desc'])   
          else:
            addMethod('', "--" + compoundOpt, dest=compoundOpt, 
                              action= _action, 
                              metavar=config[section][option]['type'],
                              default=config[section][option]['default'],
                              help=config[section][option]['desc'])   
                           
    def __add_options(self):
        if self.__withConfig:
            self.add_option("-c", "--config", dest='config', 
                action='store', default=self.__defaultConfig, 
                metavar='config_file',
                help="Full path to configuration file.")

        self.add_option("", "--verbose-help", 
            action='help', default=None, 
            metavar='flag',
            help="Display verbose help information.")
        
        self.add_option("-v", "--version", 
            action='version', default=None, 
            metavar='flag',
            help="Display version information.")
        
        self.version = self.__version
  
        if len(self._mySections) > 1:
            for section in self._mySections:
                group = OptionGroup(self, section)
                for option in self._configDef[section]:
                    compoundOpt = self.__splice_compound(section, option)
                    self.__add_option(self._configDef, compoundOpt, section, 
                                      option, group)
                self.add_option_group(group)
        else:
            for section in self._mySections:
                for option in self._configDef[section]:
                    compoundOpt = self.__splice_compound(section, option)
                    self.__add_option(self._configDef, compoundOpt, section, 
                                      option)
                    
    def __build_dict(self):
        if self.__withConfig:
            self._dict['config'] = str(getattr(self.__parsedOptions, 'config'))
        for compoundOption in dir(self.__parsedOptions):
            if compoundOption in self.__compoundOpts:
                (section, option) = self.__split_compound(compoundOption)
                if not self._dict.has_key(section):
                    self._dict[section] = {}
                
                if getattr(self.__parsedOptions, compoundOption):
                    _attr = getattr(self.__parsedOptions, compoundOption)
                    # when we have multi-valued parameters passed separately
                    # from command line, python optparser pushes them into a
                    # list. So converting all such lists to strings
                    if type(_attr) == type([]):
                      import string
                      _attr = string.join(_attr,',')
                    self._dict[section][option] = _attr
                    
        for section in self._configDef:
            for option in self._configDef[section]: 
                if self._configDef[section][option]['type'] == 'bool':
                    compoundOption = self.__splice_compound(section, option)
                    if not self._dict.has_key(section):
                        self._dict[section] = {}
                    
                    if option not in self._dict[section]:
                        self._dict[section][option] = False
 
    def __set_display_groups(self):
        if not '--verbose-help' in sys.argv:
            self.option_groups = self.__display_grps
            self.option_list = self.__display_option_list
            for group in self.option_groups:
                group.option_list = self.__display_grp_lists[group.title]
 
    def __unset_display_groups(self):
        if not '--verbose-help' in sys.argv:
            self.option_groups = self.__orig_grps
            self.option_list = self.__orig_option_list
            for group in self.option_groups:
                group.option_list = self.__orig_grp_lists[group.title]      
 
    def print_help(self, file=None):
        self.__set_display_groups()
        OptionParser.print_help(self, file)
        self.__unset_display_groups()

    def print_options(self):
        _usage = self.usage
        self.set_usage('')
        self.print_help()
        self.set_usage(_usage)
                        
    def verify(self):
        return baseConfig.verify(self)

    def replace_escape_seqs(self):
      replace_escapes(self)