AMBARI-1724. Agent has it hard-coded that HDP repo file can only be downloaded once. (smohanty)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/trunk@1461706 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -529,6 +529,9 @@ Trunk (unreleased changes):
 
  BUG FIXES
 
+ AMBARI-1724. Agent has it hard-coded that HDP repo file can only be 
+ downloaded once. (smohanty)
+
  AMBARI-1715. Ambari Agent Unit Test Failure: TestFileUtil.py. (smohanty)
 
  AMBARI-1533. Add Nagios check for ambari-agent process for each host in 

ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py

@@ -73,15 +73,23 @@ class PuppetExecutor:
       logger.info("Using default puppet on the host : " + puppetbin 
                   + " does not exist.")
       return "puppet"
-     
-  def deployRepos(self, command, tmpDir, modulesdir, taskId):
+
+  def discardInstalledRepos(self):
+    """
+    Makes agent to forget about installed repos.
+    So the next call of generate_repo_manifests() will definitely
+    install repos again
+    """
+    self.reposInstalled = False
+
+  def generate_repo_manifests(self, command, tmpDir, modulesdir, taskId):
     # Hack to only create the repo files once
-    result = []
+    manifest_list = []
     if not self.reposInstalled:
       repoInstaller = RepoInstaller(command, tmpDir, modulesdir, taskId, self.config)
-      result = repoInstaller.installRepos()
-    return result
-  
+      manifest_list = repoInstaller.generate_repo_manifests()
+    return manifest_list
+
   def puppetCommand(self, sitepp):
     modules = self.puppetModule
     puppetcommand = [self.getPuppetBinary(), "apply", "--confdir=" + modules, "--detailed-exitcodes", sitepp]
@@ -109,31 +117,16 @@ class PuppetExecutor:
   def isSuccessfull(self, returncode):
     return not self.last_puppet_has_been_killed and (returncode == 0 or returncode == 2)
 
-  def just_run_one_file(self, command, file, tmpout, tmperr):
-    result = {}
-    taskId = 0
-    if command.has_key("taskId"):
-      taskId = command['taskId']
-    #Install repos
-    self.deployRepos(command, self.tmpDir, self.modulesdir, taskId)
-    puppetEnv = os.environ
-    self.runPuppetFile(file, result, puppetEnv, tmpout, tmperr)
-    if self.isSuccessfull(result["exitcode"]):
-      # Check if all the repos were installed or not and reset the flag
-      self.reposInstalled = True
-    return result
-
-  def runCommand(self, command, tmpoutfile, tmperrfile):
+  def run_manifest(self, command, file, tmpoutfile, tmperrfile):
     result = {}
     taskId = 0
     if command.has_key("taskId"):
       taskId = command['taskId']
     puppetEnv = os.environ
     #Install repos
-    puppetFiles = self.deployRepos(command, self.tmpDir, self.modulesdir, taskId)
-    siteppFileName = os.path.join(self.tmpDir, "site-" + str(taskId) + ".pp") 
-    puppetFiles.append(siteppFileName)
-    generateManifest(command, siteppFileName, self.modulesdir, self.config)
+    repo_manifest_list = self.generate_repo_manifests(command, self.tmpDir, self.modulesdir, taskId)
+    puppetFiles = list(repo_manifest_list)
+    puppetFiles.append(file)
     #Run all puppet commands, from manifest generator and for repos installation
     #Appending outputs and errors, exitcode - maximal from all
     for puppetFile in puppetFiles:
@@ -145,10 +138,19 @@ class PuppetExecutor:
     if self.isSuccessfull(result["exitcode"]):
       # Check if all the repos were installed or not and reset the flag
       self.reposInstalled = True
-      
+
     logger.info("ExitCode : "  + str(result["exitcode"]))
     return result
 
+  def runCommand(self, command, tmpoutfile, tmperrfile):
+    taskId = 0
+    if command.has_key("taskId"):
+      taskId = command['taskId']
+    siteppFileName = os.path.join(self.tmpDir, "site-" + str(taskId) + ".pp")
+    generateManifest(command, siteppFileName, self.modulesdir, self.config)
+    result = self.run_manifest(command, siteppFileName, tmpoutfile, tmperrfile)
+    return result
+
   def runPuppetFile(self, puppetFile, result, puppetEnv, tmpoutfile, tmperrfile):
     """ Run the command and make sure the output gets propagated"""
     puppetcommand = self.puppetCommand(puppetFile)

ambari-agent/src/main/python/ambari_agent/RepoInstaller.py

@@ -89,7 +89,7 @@ class RepoInstaller:
 
     return repoPuppetFiles
 
-  def installRepos(self):
+  def generate_repo_manifests(self):
     self.prepareReposInfo()
     repoPuppetFiles = self.generateFiles()
     return repoPuppetFiles
@@ -102,7 +102,7 @@ def main():
   jsonStr = jsonFile.read() 
   parsedJson = json.loads(jsonStr)
   repoInstaller = RepoInstaller(parsedJson, '/tmp', '/home/centos/ambari_ws/ambari-agent/src/main/puppet/modules',0)
-  repoInstaller.installRepos()
+  repoInstaller.generate_repo_manifests()
   
 if __name__ == '__main__':
   main()

ambari-agent/src/main/python/ambari_agent/UpgradeExecutor.py

@@ -95,6 +95,8 @@ class UpgradeExecutor:
           'stdout'   : '',
           'stderr'   : ''
         }
+        # Request repos update (will be executed once before running any pp file)
+        self.puppetExecutor.discardInstalledRepos()
         for dir in self.SCRIPT_DIRS:
           if result['exitcode'] != 0:
             break
@@ -181,7 +183,7 @@ class UpgradeExecutor:
       filepath = os.path.join(dirpath, filename)
       if filename.endswith(".pp"):
         logger.info("Running puppet file %s" % filepath)
-        result = self.puppetExecutor.just_run_one_file(command, filepath,
+        result = self.puppetExecutor.run_manifest(command, filepath,
                                                                 tmpout, tmperr)
       elif filename.endswith(".py"):
         logger.info("Running python file %s" % filepath)

ambari-agent/src/test/python/TestRepoInstaller.py

@@ -45,8 +45,8 @@ class TestRepoInstaller(TestCase):
   @patch.object(RepoInstaller, 'prepareReposInfo')
   @patch.object(RepoInstaller, 'generateFiles')
   def testInstallRepos(self, generateFilesMock, prepareReposInfoMock):
-    result = self.repoInstaller.installRepos()
+    result = self.repoInstaller.generate_repo_manifests()
     self.assertTrue(prepareReposInfoMock.called)
     self.assertTrue(generateFilesMock.called)
-    print('installRepos result: ' + result.__str__())
+    print('generate_repo_manifests result: ' + result.__str__())
     pass

ambari-agent/src/test/python/TestUpgradeExecutor.py

@@ -39,8 +39,9 @@ class TestUpgradeExecutor(TestCase):
   @patch.object(StackVersionsFileHandler, 'write_stack_version')
   @patch('os.path.isdir')
   def test_perform_stack_upgrade(self, isdir_method, write_stack_version_method):
+    puppetExecutor = MagicMock()
     executor = UpgradeExecutor.UpgradeExecutor('pythonExecutor',
-      'puppetExecutor', AmbariConfig.AmbariConfig().getConfig())
+      puppetExecutor, AmbariConfig.AmbariConfig().getConfig())
 
     # Checking matching versions
     command = {
@@ -171,7 +172,7 @@ class TestUpgradeExecutor(TestCase):
        'stdout'   : "stdout - six.py",
        'stderr'   : "stderr - six.py"},
     ]
-    puppetExecutor.just_run_one_file.side_effect = [
+    puppetExecutor.run_manifest.side_effect = [
       {'exitcode' : 0,
        'stdout'   : "stdout - second.pp",
        'stderr'   : "stderr - second.pp"},
@@ -202,7 +203,7 @@ class TestUpgradeExecutor(TestCase):
        'stdout'   : "stdout - python.py",
        'stderr'   : "stderr - python.py"},
     ]
-    puppetExecutor.just_run_one_file.side_effect = [
+    puppetExecutor.run_manifest.side_effect = [
       {'exitcode' : 0,
        'stdout'   : "stdout - puppet.pp",
        'stderr'   : "stderr - puppet.pp"},

ambari-agent/src/test/python/examples/ControllerTester.py

@@ -92,9 +92,9 @@ responseId = Int(0)
 def main():
 
   if disable_python_and_puppet:
-    with patch.object(PuppetExecutor.PuppetExecutor, 'just_run_one_file') \
-                                          as just_run_one_file_pp_method:
-      just_run_one_file_pp_method.side_effect = \
+    with patch.object(PuppetExecutor.PuppetExecutor, 'run_manifest') \
+                                          as run_manifest_method:
+      run_manifest_method.side_effect = \
               lambda command, file, tmpout, tmperr: {
           'exitcode' : 0,
           'stdout'   : "Simulated run of pp %s" % file,