ソースを参照

HADOOP-2855. Fixes the way HOD handles relative paths for file and directory options. Contributed by Vinod Kumar Vavilapalli.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@643297 13f79535-47bb-0310-9956-ffa450edef68
Hemanth Yamijala 17 年 前
コミット
8394e6f29a

+ 96 - 0
src/contrib/hod/CHANGES.txt

@@ -0,0 +1,96 @@
+HOD Change Log
+
+
+Trunk (unreleased changes)
+
+  IMPROVEMENTS
+
+    HADOOP-2775.  Adds unit test framework for HOD.
+    (Vinod Kumar Vavilapalli via ddas).
+
+    HADOOP-2848. [HOD]hod -o list and deallocate works even after deleting
+    the cluster directory. (Hemanth Yamijala via ddas)
+
+    HADOOP-2899. [HOD] Cleans up hdfs:///mapredsystem directory after
+    deallocation. (Hemanth Yamijala via ddas)
+
+    HADOOP-2796. Enables distinguishing exit codes from user code vis-a-vis
+    HOD's exit code. (Hemanth Yamijala via ddas)
+
+  BUG FIXES
+
+    HADOOP-2924. Fixes an address problem to do with TaskTracker binding
+    to an address. (Vinod Kumar Vavilapalli via ddas)
+
+    HADOOP-2970. Fixes a problem to do with Wrong class definition for
+    hodlib/Hod/hod.py for Python < 2.5.1.
+    (Vinod Kumar Vavilapalli via ddas)
+
+    HADOOP-2783. Fixes a problem to do with import in
+    hod/hodlib/Common/xmlrpc.py. (Vinod Kumar Vavilapalli via ddas)
+
+    HADOOP-2936. Fixes HOD in a way that it generates hdfs://host:port on the
+    client side configs. (Vinod Kumar Vavilapalli via ddas)
+
+    HADOOP-2983. [HOD] Fixes the problem - local_fqdn() returns None when
+    gethostbyname_ex doesnt return any FQDNs. (Craig Macdonald via ddas)
+
+    HADOOP-2982. Fixes a problem in the way HOD looks for free nodes.
+    (Hemanth Yamijala via ddas)
+
+    HADOOP-2855. Fixes the way HOD handles relative paths for cluster
+    directory, script file and other options.
+    (Vinod Kumar Vavilapalli via yhemanth)
+
+Release 0.16.2 - 2008-04-02
+
+  BUG FIXES
+
+    HADOOP-3103. [HOD] Hadoop.tmp.dir should not be set to cluster
+    directory. (Vinod Kumar Vavilapalli via ddas).
+
+Release 0.16.1 - 2008-03-13
+
+  INCOMPATIBLE CHANGES
+
+    HADOOP-2861. Improve the user interface for the HOD commands.
+    Command line structure has changed. (Hemanth Yamijala via nigel)
+
+  IMPROVEMENTS
+
+    HADOOP-2730. HOD documentation update.
+    (Vinod Kumar Vavilapalli via ddas)
+
+    HADOOP-2911. Make the information printed by the HOD allocate and
+    info commands less verbose and clearer. (Vinod Kumar via nigel)
+
+  BUG FIXES
+
+    HADOOP-2766. Enables setting of HADOOP_OPTS env variable for the hadoop
+    daemons through HOD. (Vinod Kumar Vavilapalli via ddas)
+
+    HADOOP-2809.  Fix HOD syslog config syslog-address so that it works.
+    (Hemanth Yamijala via nigel)
+
+    HADOOP-2847.  Ensure idle cluster cleanup works even if the JobTracker
+    becomes unresponsive to RPC calls. (Hemanth Yamijala via nigel)
+
+    HADOOP-2925. Fix HOD to create the mapred system directory using a
+    naming convention that will avoid clashes in multi-user shared
+    cluster scenario. (Hemanth Yamijala via nigel)
+
+Release 0.16.0 - 2008-02-07
+
+  NEW FEATURES
+
+    HADOOP-1301.  Hadoop-On-Demand (HOD): resource management
+    provisioning for Hadoop. (Hemanth Yamijala via nigel)
+
+  BUG FIXES
+
+    HADOOP-2720. Jumbo bug fix patch to HOD.  Final sync of Apache SVN with
+    internal Yahoo SVN.  (Hemanth Yamijala via nigel)
+
+    HADOOP-2740. Fix HOD to work with the configuration variables changed in
+    HADOOP-2404. (Hemanth Yamijala via omalley)
+

+ 1 - 1
src/contrib/hod/bin/hod

@@ -211,7 +211,7 @@ defList = { 'hod' : (
              ('svcrgy-addr', 'address', 'Download HTTP address.',
               False, None, False, False),             
              
-             ('hadoop-tar-ball', 'string', 'hadoop program tar ball.',
+             ('hadoop-tar-ball', 'uri', 'hadoop program tar ball.',
               True, None, False, False, 't'),
 
              ('max-connect','pos_int','max connections allowed for a single tarball server',

+ 1 - 1
src/contrib/hod/bin/ringmaster

@@ -101,7 +101,7 @@ defList = { 'ringmaster' : (
              ('svcrgy-addr', 'address', 'Download HTTP address.',
               False, None, False, True),             
              
-             ('hadoop-tar-ball', 'string', 'hadoop program tar ball.',
+             ('hadoop-tar-ball', 'uri', 'hadoop program tar ball.',
               False, None, False, False),
         
              ('max-connect','pos_int','max connections allowed for a single tarball server',

+ 4 - 5
src/contrib/hod/hodlib/Common/setup.py

@@ -315,7 +315,7 @@ class baseConfig:
 	
         if self._configDef:
             errorCount = 0
-            configValidator = typeValidator()
+            configValidator = typeValidator(self.__originalDir)
 
             # foreach section and option by type string as defined in configDef
             #   add value to be validated to validator
@@ -330,10 +330,8 @@ class baseConfig:
                                 configValidator.add(configVarName,
                                     self._configDef[section][option]['type'],
                                     self._dict[section][option])
-                            elif self._configDef[section][option]['type'] \
-                                != 'file' and \
-                                self._configDef[section][option]['type'] != \
-                                'directory':
+                            else:
+                                # If asked not to validate, just normalize
                                 self[section][option] = \
                                     configValidator.normalize(
                                     self._configDef[section][option]['type'], 
@@ -351,6 +349,7 @@ class baseConfig:
                                     self._configDef[section][option]['default']
                                     )
                         else:        
+                            # This should not happen. Just in case, take this as 'to be validated' case.
                             configValidator.add(configVarName,
                                 self._configDef[section][option]['type'],
                                 self._dict[section][option])

+ 60 - 5
src/contrib/hod/hodlib/Common/types.py

@@ -59,9 +59,10 @@
       string         - arbitrarily long string
       list           - comma seperated list of strings of arbitrary length,
       keyval         - comma seperated list of key=value pairs, key does not 
-                       need to be unique."""
+                       need to be unique.
+      uri            - a uri """
 
-import sys, os, socket, pwd, grp, stat, re, re, string, pprint
+import sys, os, socket, pwd, grp, stat, re, re, string, pprint, urlparse
 
 from tcp import tcpSocket, check_net_address, check_ip_address
 from util import check_timestamp
@@ -143,6 +144,9 @@ types = { 'directory'      : { 'db'    : 'string',
 
           'keyval'         : { 'db'    : 'string',
                                'units' : None     },
+          
+          'uri'            : { 'db'    : 'string',
+                               'units' : None     },
 
           ''               : { 'db'    : 'string',
                                'units' : None     }}
@@ -376,6 +380,9 @@ class typeToString:
     def __tostring_user_group(self, value):
         return self.__tostring(value)
 
+    def __tostring_uri(self, value):
+        return self.__tostring(value)
+
     def __tostring_nothing(self, value):
         return value
 
@@ -383,12 +390,13 @@ class typeValidator:
     """Type validation class used to normalize values or validated 
        single/large sets of values by type."""
 
-    def __init__(self):
+    def __init__(self, originalDir=None):
         self.verifyFunctions = {}
         self.__build_verify_functions()
 
         self.validateList = []
         self.validatedInfo = []
+        self.__originalDir = originalDir
 
     def __getattr__(self, attrname):
         """validateList  = [ { 'func' : <bound method configValidator>,
@@ -472,7 +480,7 @@ class typeValidator:
         return valueInfo
       
     def __norm_directory(self, value):
-        return os.path.realpath(value)
+        return self.__normalizedPath(value)
 
     def __verify_address(self, type, value):
         valueInfo = self.__get_value_info()
@@ -720,7 +728,7 @@ class typeValidator:
         return valueInfo
       
     def __norm_file(self, value):
-        return os.path.realpath(value)
+        return self.__normalizedPath(value)
 
     def __verify_size(self, type, value):
         valueInfo = self.__get_value_info()
@@ -846,6 +854,44 @@ class typeValidator:
             
         return (start, end)     
 
+    def __verify_uri(self, type, value):
+        valueInfo = self.__get_value_info()
+
+        _norm = None
+        try:
+            uriComponents = urlparse.urlparse(value)
+            if uriComponents[0] == '' or uriComponents[0] == 'file':
+              # if scheme is '' or 'file'
+              if not os.path.isfile(uriComponents[2]) and \
+                                         not os.path.isdir(uriComponents[2]):
+                  raise Exception("Invalid local URI")
+              else:
+                  self.__set_value_info(valueInfo, normalized=self.normalize(
+                                                                  type,value))
+            else:
+              # other schemes
+              # currently not checking anything. TODO
+              self.__set_value_info(valueInfo, normalized=self.normalize(
+                                                                   type,value))
+        except:
+            errorString = "%s is an invalid uri" % value
+            self.__set_value_info(valueInfo, errorData=errorString)
+
+        return valueInfo
+
+    def __norm_uri(self, value):
+       uriComponents = list(urlparse.urlparse(value))
+       if uriComponents[0] == '':
+          # if scheme is '''
+          return self.__normalizedPath(uriComponents[2])
+       elif uriComponents[0] == 'file':
+          # if scheme is 'file'
+          normalizedPath = self.__normalizedPath(uriComponents[2])
+          return urlparse.urlunsplit(uriComponents[0:1] + [normalizedPath] + uriComponents[3:])
+
+       # Not dealing with any other case right now
+       return value
+
     def __verify_timestamp(self, type, value):
         valueInfo = self.__get_value_info()
 
@@ -990,6 +1036,15 @@ class typeValidator:
                 raise Exception("\nMissing a return value: valueInfo\n%s" % \
                     self.verifyFunctions[valItem['type']](valItem['value']))
 
+    def __normalizedPath(self, value):    
+        oldWd = os.getcwd()
+        if self.__originalDir:
+          os.chdir(self.__originalDir)
+        normPath = os.path.realpath(value)
+        os.chdir(oldWd)
+        return normPath
+
+
 class display:
     def __init__(self):
         self.displayFunctions = {}

+ 180 - 0
src/contrib/hod/testing/testTypes.py

@@ -0,0 +1,180 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, os, sys, re, threading, time
+
+myDirectory = os.path.realpath(sys.argv[0])
+rootDirectory   = re.sub("/testing/.*", "", myDirectory)
+
+sys.path.append(rootDirectory)
+
+from testing.lib import BaseTestSuite
+
+excludes = ['']
+
+import tempfile, shutil, getpass, random
+from hodlib.Common.types import typeValidator
+
+# All test-case classes should have the naming convention test_.*
+class test_typeValidator(unittest.TestCase):
+  def setUp(self):
+    self.originalDir = os.getcwd()
+    self.validator = typeValidator(self.originalDir)
+    self.tempDir = tempfile.mkdtemp(dir='/tmp/hod-%s' % getpass.getuser(),
+                                    prefix='test_Types_typeValidator_tempDir')
+    self.tempFile = tempfile.NamedTemporaryFile(dir=self.tempDir)
+
+    # verification : error strings
+    self.errorStringsForVerify = {
+                          'pos_int' : 0,
+                          'uri' : '%s is an invalid uri',
+                          'directory' : 0,
+                          'file' : 0,
+                        }
+
+    # verification : valid vals
+    self.verifyValidVals = [
+                            ('pos_int', 0),
+                            ('pos_int', 1),
+                            ('directory', self.tempDir),
+                            ('directory', '/tmp/hod-%s/../../%s' % \
+                                    (getpass.getuser(), self.tempDir)),
+                            ('file', self.tempFile.name),
+                            ('file', '/tmp/hod-%s/../../%s' % \
+                                    (getpass.getuser(), self.tempFile.name)),
+                            ('uri', 'file://localhost/' + self.tempDir),
+                            ('uri', 'file:///' + self.tempDir),
+                            ('uri', 'file:///tmp/hod-%s/../../%s' % \
+                                    (getpass.getuser(), self.tempDir)),
+                            ('uri', 'file://localhost/tmp/hod-%s/../../%s' % \
+                                    (getpass.getuser(), self.tempDir)),
+                            ('uri', 'http://hadoop.apache.org/core/'),
+                            ('uri', self.tempDir),
+                            ('uri', '/tmp/hod-%s/../../%s' % \
+                                    (getpass.getuser(), self.tempDir)),
+                           ]
+
+    # generate an invalid uri
+    randomNum = random.random()
+    while os.path.exists('/%s' % randomNum):
+      # Just to be sure :)
+      randomNum = random.random()
+    invalidUri = 'file://localhost/%s' % randomNum
+
+    # verification : invalid vals
+    self.verifyInvalidVals = [
+                              ('pos_int', -1),
+                              ('uri', invalidUri),
+                              ('directory', self.tempFile.name),
+                              ('file', self.tempDir),
+                             ]
+
+    # normalization : vals
+    self.normalizeVals = [
+                            ('pos_int', 1, 1),
+                            ('pos_int', '1', 1),
+                            ('directory', self.tempDir, self.tempDir),
+                            ('directory', '/tmp/hod-%s/../../%s' % \
+                                  (getpass.getuser(), self.tempDir), 
+                                                      self.tempDir),
+                            ('file', self.tempFile.name, self.tempFile.name),
+                            ('file', '/tmp/hod-%s/../../%s' % \
+                                    (getpass.getuser(), self.tempFile.name),
+                                                         self.tempFile.name),
+                            ('uri', 'file://localhost' + self.tempDir, 
+                                  'file://' + self.tempDir),
+                            ('uri', 'file://127.0.0.1' + self.tempDir, 
+                                  'file://' + self.tempDir),
+                            ('uri', 'http://hadoop.apache.org/core',
+                                  'http://hadoop.apache.org/core'),
+                            ('uri', self.tempDir, self.tempDir),
+                            ('uri', '/tmp/hod-%s/../../%s' % \
+                                  (getpass.getuser(), self.tempDir), 
+                                                      self.tempDir),
+                         ]
+    pass
+
+  # All testMethods have to have their names start with 'test'
+  def testnormalize(self):
+    for (type, originalVal, normalizedVal) in self.normalizeVals:
+      # print type, originalVal, normalizedVal,\
+      #                          self.validator.normalize(type, originalVal)
+      assert(self.validator.normalize(type, originalVal) == normalizedVal)
+    pass
+
+  def test__normalize(self):
+    # Special test for functionality of private method __normalizedPath
+    tmpdir = tempfile.mkdtemp(dir=self.originalDir) #create in self.originalDir
+    oldWd = os.getcwd()
+    os.chdir('/')
+    tmpdirName = re.sub(".*/","",tmpdir)
+    # print re.sub(".*/","",tmpdirName)
+    # print os.path.join(self.originalDir,tmpdir)
+    (type, originalVal, normalizedVal) = \
+                                    ('file', tmpdirName, \
+                                    os.path.join(self.originalDir,tmpdirName))
+    assert(self.validator.normalize(type, originalVal) == normalizedVal)
+    os.chdir(oldWd)
+    os.rmdir(tmpdir)
+    pass
+    
+  def testverify(self):
+    # test verify method
+
+    # test valid vals
+    for (type,value) in self.verifyValidVals:
+      valueInfo = { 'isValid' : 0, 'normalized' : 0, 'errorData' : 0 }
+      valueInfo = self.validator.verify(type,value)
+      # print type, value, valueInfo
+      assert(valueInfo['isValid'] == 1)
+
+    # test invalid vals
+    for (type,value) in self.verifyInvalidVals:
+      valueInfo = { 'isValid' : 0, 'normalized' : 0, 'errorData' : 0 }
+      valueInfo = self.validator.verify(type,value)
+      # print type, value, valueInfo
+      assert(valueInfo['isValid'] == 0)
+      if valueInfo['errorData'] != 0:
+        # if there is any errorData, check
+        assert(valueInfo['errorData'] == \
+                                      self.errorStringsForVerify[type] % value)
+
+    pass
+
+  def tearDown(self):
+    self.tempFile.close()
+    if os.path.exists(self.tempDir):
+      shutil.rmtree(self.tempDir)
+    pass
+
+class TypesTestSuite(BaseTestSuite):
+  def __init__(self):
+    # suite setup
+    BaseTestSuite.__init__(self, __name__, excludes)
+    pass
+  
+  def cleanUp(self):
+    # suite tearDown
+    pass
+
+def RunTypesTests():
+  # modulename_suite
+  suite = TypesTestSuite()
+  testResult = suite.runTests()
+  suite.cleanUp()
+  return testResult
+
+if __name__ == "__main__":
+  RunTypesTests()