HADOOP-2775. Adds unit test framework for HOD. Contributed by Vinod Kumar Vavilapalli.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@636995 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -92,6 +92,9 @@ Trunk (unreleased changes)
     HADOOP-2981.  Update README.txt to reflect the upcoming use of
     cryptography. (omalley)
 
+    HADOOP-2775.  Adds unit test framework for HOD. 
+    (Vinod Kumar Vavilapalli via ddas).
+
   OPTIMIZATIONS
 
     HADOOP-2790.  Fixed inefficient method hasSpeculativeTask by removing

src/contrib/hod/build.xml

@@ -32,4 +32,45 @@
         </copy>
         <chmod dir="${dist.dir}/contrib/${name}/bin" perm="a+x" includes="*"/>
     </target>
+
+    <target name="test" depends="compile" description="Run HOD unit tests">  
+      <antcall target="python.pathcheck"/>
+      <antcall target="checkAndRunTests"/>
+    </target>
+
+    <target name="checkAndRunTests" if="python.home">
+      <!-- Check python version now -->
+      <exec executable="/bin/sh" outputproperty="hodtest.pythonVersion">
+          <arg value="-c" />
+          <arg value="${python.home}/python -V" />
+      </exec>
+      <condition property="python.versionmatched">
+        <!--- Currently check for only 2.5.1 -->
+        <equals arg1="${hodtest.pythonVersion}" arg2="Python 2.5.1" />
+      </condition>
+      <antcall target="python.versioncheck"/>
+      <antcall target="runtests"/>
+    </target>
+ 
+    <target name="python.pathcheck" unless="python.home">
+      <echo message="'python.home' is not defined. Please pass -Dpython.home=&lt;Path to Python&gt; to Ant on the command-line."/>
+    </target>
+
+    <target name="runtests" if="python.versionmatched">
+      <echo message="Using Python at : ${python.home}" />
+      <echo message="Version : ${hodtest.pythonVersion}"/>
+      <exec executable="/bin/sh" resultproperty="hodtest.failedTests">
+        <arg value="-c" />
+          <arg value="${python.home}/python ${build.dir}/testing/main.py" />
+      </exec>
+      <condition property="hodtest.success">
+        <equals arg1="${hodtest.failedTests}" arg2="0"/>
+      </condition>
+      <fail message="TestCases failed. ${hodtest.failedTests} failed to run successfully." unless="hodtest.success"/>
+    </target>
+    
+    <target name="python.versioncheck" unless="python.versionmatched">
+      <echo message="Need Python version 2.5.1. You specified ${hodtest.pythonVersion}"/>
+    </target>
+
 </project>

src/contrib/hod/testing/__init__.py

@@ -0,0 +1,15 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.

src/contrib/hod/testing/lib.py

@@ -0,0 +1,64 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, re, sys
+
+class BaseTestSuite():
+  def __init__(self, name, excludes):
+    self.name = name
+    self.excludes = excludes
+    pass
+  
+  def runTests(self):
+    # Create a runner
+    self.runner = unittest.TextTestRunner()
+    
+    # Get all the test-case classes
+    # From module import *
+    mod = __import__(self.name, fromlist=['*'])
+    modItemsList = dir(mod)
+
+    allsuites = []
+
+    # Create all the test suites
+    for modItem in modItemsList:
+      if re.search(r"^test_", modItem):
+        # Yes this is a test class
+        if modItem not in self.excludes:
+          test_class = getattr(mod, modItem)
+          allsuites.append(unittest.makeSuite(test_class))
+
+    # Create a master suite to be run.
+    alltests = unittest.TestSuite(tuple(allsuites))
+
+    # Run the master test suite.
+    runner = self.runner.run(alltests)
+    if(runner.wasSuccessful()): return 0
+    printLine( "%s test(s) failed." % runner.failures.__len__())
+    printLine( "%s test(s) threw errors." % runner.errors.__len__())
+    return runner.failures.__len__() + runner.errors.__len__()
+
+  def cleanUp(self):
+    # suite tearDown
+    pass
+
+def printLine(str):
+  print >>sys.stderr, str
+
+def printSeparator():
+  str = ""
+  for i in range(0,79):
+    str = str + "*"
+  print >>sys.stderr, "\n", str, "\n"

src/contrib/hod/testing/main.py

@@ -0,0 +1,84 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, os, sys, re
+
+myPath = os.path.realpath(sys.argv[0])
+rootDirectory   = re.sub("/testing/.*", "", myPath)
+testingDir = os.path.join(rootDirectory, "testing")
+
+sys.path.append(rootDirectory)
+
+from testing.lib import printSeparator, printLine
+
+moduleList = []
+allList = []
+excludes = [
+            'Xmlrpc'
+           ]
+
+# Build a module list by scanning through all files in testingDir
+for file in os.listdir(testingDir):
+  if(re.search(r".py$", file) and re.search(r"^test", file)):
+    # All .py files with names starting in 'test'
+    module = re.sub(r"^test","",file)
+    module = re.sub(r".py$","",module)
+    allList.append(module)
+    if module not in excludes:
+      moduleList.append(module)
+
+printLine("All testcases - %s" % allList)
+printLine("Excluding the testcases - %s" % excludes)
+printLine("Executing the testcases - %s" % moduleList)
+
+testsResult = 0
+# Now import each of these modules and start calling the corresponding
+#testSuite methods
+for moduleBaseName in moduleList:
+  try:
+    module = "testing.test" + moduleBaseName
+    suiteCaller = "Run" + moduleBaseName + "Tests"
+    printSeparator()
+    printLine("Running %s" % suiteCaller)
+
+    # Import the corresponding test cases module
+    imported_module = __import__(module , fromlist=[suiteCaller] )
+    
+    # Call the corresponding suite method now
+    testRes = getattr(imported_module, suiteCaller)()
+    testsResult = testsResult + testRes
+    printLine("Finished %s. TestSuite Result : %s\n" % \
+                                              (suiteCaller, testRes))
+  except ImportError, i:
+    # Failed to import a test module
+    printLine(i)
+    testsResult = testsResult + 1
+    pass
+  except AttributeError, n:
+    # Failed to get suiteCaller from a test module
+    printLine(n)
+    testsResult = testsResult + 1
+    pass
+  except Exception, e:
+    # Test module suiteCaller threw some exception
+    printLine("%s failed. \nReason : %s" % (suiteCaller, e))
+    printLine("Skipping %s" % suiteCaller)
+    testsResult = testsResult + 1
+    pass
+
+if testsResult != 0:
+  printSeparator()
+  printLine("Total testcases with failure or error : %s" % testsResult)
+sys.exit(testsResult)

src/contrib/hod/testing/testModule.py

@@ -0,0 +1,88 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, os, sys, re, threading, time
+
+myDirectory = os.path.realpath(sys.argv[0])
+rootDirectory   = re.sub("/testing/.*", "", myDirectory)
+
+sys.path.append(rootDirectory)
+
+from testing.lib import BaseTestSuite
+
+excludes = ['test_MINITEST3']
+
+# All test-case classes should have the naming convention test_.*
+class test_MINITEST1(unittest.TestCase):
+  def setUp(self):
+    pass
+
+  # All testMethods have to have their names start with 'test'
+  def testSuccess(self):
+    pass
+    
+  def testFailure(self):
+    pass
+
+  def tearDown(self):
+    pass
+
+class test_MINITEST2(unittest.TestCase):
+  def setUp(self):
+    pass
+
+  # All testMethods have to have their names start with 'test'
+  def testSuccess(self):
+    pass
+    
+  def testFailure(self):
+    pass
+
+  def tearDown(self):
+    pass
+
+class test_MINITEST3(unittest.TestCase):
+  def setUp(self):
+    pass
+
+  # All testMethods have to have their names start with 'test'
+  def testSuccess(self):
+    pass
+    
+  def testFailure(self):
+    pass
+
+  def tearDown(self):
+    pass
+
+class ModuleTestSuite(BaseTestSuite):
+  def __init__(self):
+    # suite setup
+    BaseTestSuite.__init__(self, __name__, excludes)
+    pass
+  
+  def cleanUp(self):
+    # suite tearDown
+    pass
+
+def RunModuleTests():
+  # modulename_suite
+  suite = ModuleTestSuite()
+  testResult = suite.runTests()
+  suite.cleanUp()
+  return testResult
+
+if __name__ == "__main__":
+  RunModuleTests()

src/contrib/hod/testing/testRingmasterRPCs.py

@@ -0,0 +1,131 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, os, sys, re, threading, time
+
+import logging
+
+myDirectory    = os.path.realpath(sys.argv[0])
+rootDirectory   = re.sub("/testing/.*", "", myDirectory)
+
+sys.path.append(rootDirectory)
+
+from testing.lib import BaseTestSuite
+
+excludes = ['test_MINITEST1', 'test_MINITEST2']
+ 
+from hodlib.GridServices import *
+from hodlib.Common.desc import ServiceDesc
+from hodlib.RingMaster.ringMaster import _LogMasterSources
+
+# All test-case classes should have the naming convention test_.*
+class test_MINITEST1(unittest.TestCase):
+  def setUp(self):
+    pass
+
+  # All testMethods have to have their names start with 'test'
+  def testSuccess(self):
+    pass
+    
+  def testFailure(self):
+    pass
+
+  def tearDown(self):
+    pass
+
+class test_MINITEST2(unittest.TestCase):
+  def setUp(self):
+    pass
+
+  # All testMethods have to have their names start with 'test'
+  def testSuccess(self):
+    pass
+    
+  def testFailure(self):
+    pass
+
+  def tearDown(self):
+    pass
+
+class test_GetCommand(unittest.TestCase):
+  def setUp(self):
+    self.config = {
+       'hod': {}, 
+      'resource_manager': {
+                            'id': 'torque', 
+                            'batch-home': '/home/y/'
+                          }, 
+       'ringmaster': {
+                      'max-connect' : 2
+                     }, 
+       'hodring': {
+                  }, 
+       'gridservice-mapred': { 
+                              'id': 'mapred' 
+                             } ,
+       'gridservice-hdfs': { 
+                              'id': 'hdfs' 
+                            }, 
+       'servicedesc' : {} ,
+       'nodepooldesc': {} , 
+       }
+
+    hdfsDesc = self.config['servicedesc']['hdfs'] = ServiceDesc(self.config['gridservice-hdfs'])
+    mrDesc = self.config['servicedesc']['mapred'] = ServiceDesc(self.config['gridservice-mapred'])
+
+    # API : serviceObj = service(desc, workDirs, reqNodes, version)
+    self.hdfs = Hdfs(hdfsDesc, [], 0, 17)
+    self.hdfsExternal = HdfsExternal(hdfsDesc, [], 17)
+    self.mr = MapReduce(mrDesc, [],1, 17)
+    self.mrExternal = MapReduceExternal(mrDesc, [], 17)
+    
+    self.log = logging.getLogger()
+    pass
+
+  # All testMethods have to have their names start with 'test'
+  def testBothInternal(self):
+    self.serviceDict = {}
+    self.serviceDict[self.hdfs.getName()] = self.hdfs
+    self.serviceDict[self.mr.getName()] = self.mr
+    self.rpcSet = _LogMasterSources(self.serviceDict, self.config, None, self.log, None)
+
+    cmdList = self.rpcSet.getCommand('localhost')
+    self.assertEquals(cmdList.__len__(), 2)
+    self.assertEquals(cmdList[0].dict['argv'][0], 'namenode')
+    self.assertEquals(cmdList[1].dict['argv'][0], 'namenode')
+    pass
+    
+  def tearDown(self):
+    pass
+
+class RingmasterRPCsTestSuite(BaseTestSuite):
+  def __init__(self):
+    # suite setup
+    BaseTestSuite.__init__(self, __name__, excludes)
+    pass
+  
+  def cleanUp(self):
+    # suite tearDown
+    pass
+
+def RunRingmasterRPCsTests():
+  # modulename_suite
+  suite = RingmasterRPCsTestSuite()
+  testResult = suite.runTests()
+  suite.cleanUp() 
+  return testResult
+
+if __name__ == "__main__":
+  RunRingmasterRPCsTests()

src/contrib/hod/testing/testXmlrpc.py

@@ -0,0 +1,99 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+import unittest, os, sys, re, threading, time
+
+myDirectory    = os.path.realpath(sys.argv[0])
+rootDirectory   = re.sub("/testing/.*", "", myDirectory)
+
+sys.path.append(rootDirectory)
+
+from hodlib.Common.xmlrpc import hodXRClient
+from hodlib.Common.socketServers import hodXMLRPCServer
+
+from testing.lib import BaseTestSuite
+
+excludes = []
+
+class test_HodXRClient(unittest.TestCase):
+  def setUp(self):
+    pass
+
+  # All testMethods have to have their names start with 'test'
+  def testSuccess(self):
+    client = hodXRClient('http://localhost:5555', retryRequests=False)
+    self.assertEqual(client.testing(), True)
+    pass
+    
+  def testFailure(self):
+    client = hodXRClient('http://localhost:5555', retryRequests=False)
+    # client.noMethod()
+    self.assertRaises(Exception, client.noMethod)
+    pass
+
+  def testTimeout(self):
+    """HOD should raise Exception when rpc call times out"""
+    client = hodXRClient('http://localhost:567823', retryRequests=False)
+    # client.noMethod()
+    self.assertRaises(Exception, client.testing)
+    pass
+
+  def testInterrupt(self):
+    """ HOD should raise HodInterruptException when interrupted"""
+    from hodlib.Common.util import hodInterrupt, HodInterruptException
+
+    def interrupt():
+      client = hodXRClient('http://localhost:59087')
+      thread = threading.Thread(name='testinterrupt', target=client.testing)
+      thread.start()
+      time.sleep(1)
+      hodInterrupt.setFlag()
+      thread.join()
+
+    self.assertRaises(HodInterruptException, interrupt)
+    pass
+
+  def tearDown(self):
+    pass
+
+class XmlrpcTestSuite(BaseTestSuite):
+  def __init__(self):
+    # suite setup
+    BaseTestSuite.__init__(self, __name__, excludes)
+
+    def rpcCall():
+      return True
+    
+    self.server = hodXMLRPCServer('localhost', ['5555'])
+    self.server.register_function(rpcCall, 'testing')
+    self.thread = threading.Thread(name="server", 
+                                   target=self.server._serve_forever)
+    self.thread.start()
+    time.sleep(1) # give some time to start server
+  
+  def cleanUp(self):
+    # suite tearDown
+    self.server.stop()
+    self.thread.join()
+
+def RunXmlrpcTests():
+  # modulename_suite
+  suite = XmlrpcTestSuite()
+  testResult = suite.runTests()
+  suite.cleanUp()
+  return testResult
+
+if __name__ == "__main__":
+  RunXmlrpcTests()