AMBARI-12682: Ambari repo file (e.g. ambari.repo) should not be required (Nahappan Somasundaram via jluniya)

ambari-common/src/main/python/ambari_commons/os_utils.py

@@ -21,11 +21,13 @@ limitations under the License.
 import os
 import shutil
 import string
-import pwd
-import stat
-
 from ambari_commons import OSCheck
 
+if OSCheck.is_windows_family():
+  pass
+else:
+  import pwd
+
 if OSCheck.is_windows_family():
   from ambari_commons.os_windows import os_change_owner, os_getpass, os_is_root, os_run_os_command, \
     os_set_open_files_limit, os_set_file_permissions
@@ -155,7 +157,8 @@ def get_ambari_repo_file_full_name():
   elif OSCheck.is_suse_family():
     ambari_repo_file = "/etc/zypp/repos.d/ambari.repo"
   elif OSCheck.is_windows_family():
-    ambari_repo_file = ""
+    ambari_repo_file = os.path.join(os.environ[ChocolateyConsts.CHOCOLATEY_INSTALL_VAR_NAME],
+                                    ChocolateyConsts.CHOCOLATEY_CONFIG_DIR, ChocolateyConsts.CHOCOLATEY_CONFIG_FILENAME)
   else:
     raise Exception('Ambari repo file path not set for current OS.')
 
@@ -163,4 +166,15 @@ def get_ambari_repo_file_full_name():
 
 # Gets the owner of the specified file
 def get_file_owner(file_full_name):
-  return pwd.getpwuid(os.stat(file_full_name).st_uid).pw_name
+  if OSCheck.is_windows_family():
+    return ""
+  else:
+      return pwd.getpwuid(os.stat(file_full_name).st_uid).pw_name
+
+#
+# Chololatey package manager constants for Windows
+#
+class ChocolateyConsts:
+  CHOCOLATEY_INSTALL_VAR_NAME = "ChocolateyInstall"
+  CHOCOLATEY_CONFIG_DIR = "config"
+  CHOCOLATEY_CONFIG_FILENAME = "chocolatey.config"

ambari-server/src/main/python/ambari_server/setupSecurity.py

@@ -171,8 +171,9 @@ def adjust_directory_permissions(ambari_user):
   ambari_repo_file = get_ambari_repo_file_full_name()
 
   if ambari_repo_file:
-    ambari_repo_file_owner = get_file_owner(ambari_repo_file)
-    configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((ambari_repo_file, "644", ambari_repo_file_owner, False))
+    if (os.path.exists(ambari_repo_file)):
+        ambari_repo_file_owner = get_file_owner(ambari_repo_file)
+        configDefaults.NR_ADJUST_OWNERSHIP_LIST.append((ambari_repo_file, "644", ambari_repo_file_owner, False))
 
 
   print "Adjusting ambari-server permissions and ownership..."

ambari-server/src/main/python/bootstrap.py

@@ -312,7 +312,7 @@ class BootstrapWindows(Bootstrap):
 
   def copyChocolateyConfig(self):
     # Copying chocolatey.config file
-    fileToCopy = os.path.join(os.environ[self.CHOCOLATEY_INSTALL_VAR_NAME], self.CHOCOLATEY_CONFIG_DIR, self.CHOCOLATEY_CONFIG_FILENAME)
+    fileToCopy = getConfigFile()
     target = os.path.join(self.getTempFolder(), self.CHOCOLATEY_CONFIG_FILENAME)
     self.host_log.write("==========================\n")
     self.host_log.write("Copying chocolatey config file...")
@@ -349,6 +349,9 @@ class BootstrapWindows(Bootstrap):
     self.host_log.write("\n")
     return retcode
 
+  def getConfigFile(self):
+    return os.path.join(os.environ[self.CHOCOLATEY_INSTALL_VAR_NAME], self.CHOCOLATEY_CONFIG_DIR, self.CHOCOLATEY_CONFIG_FILENAME)
+
   def run(self):
     """ Copy files and run commands on remote host """
     self.status["start_time"] = time.time()
@@ -360,15 +363,25 @@ class BootstrapWindows(Bootstrap):
                     self.configureChocolatey,
                     self.runSetupAgent
     ]
-    # Execution of action queue
+
     last_retcode = 0
-    while action_queue and last_retcode == 0:
-      action = action_queue.pop(0)
-      ret = self.try_to_execute(action)
+
+    if os.path.exists(getConfigFile()) :
+    # Checking execution result   # Execution of action queue
+      while action_queue and last_retcode == 0:
+        action = action_queue.pop(0)
+        ret = self.try_to_execute(action)
+        last_retcode = ret["exitstatus"]
+        err_msg = ret["errormsg"]
+        std_out = ret["log"]
+    else:
+      # If config file is not found, then assume that the hosts have
+      # already been provisioned. Attempt to run the setupAgent script alone.
+      ret = self.try_to_execute(self.runSetupAgent)
       last_retcode = ret["exitstatus"]
       err_msg = ret["errormsg"]
       std_out = ret["log"]
-    # Checking execution result
+      pass
     if last_retcode != 0:
       message = "ERROR: Bootstrap of host {0} fails because previous action " \
                 "finished with non-zero exit code ({1})\nERROR MESSAGE: {2}\nSTDOUT: {3}".format(self.host, last_retcode, err_msg, std_out)
@@ -484,47 +497,59 @@ class BootstrapDefault(Bootstrap):
     return "sudo chmod 644 {0}".format(self.getRepoFile())
 
   def copyNeededFiles(self):
+    # get the params
+    params = self.shared_state
+
     # Copying the files
     fileToCopy = self.getRepoFile()
     target = self.getRemoteName(self.AMBARI_REPO_FILENAME)
 
-    self.host_log.write("==========================\n")
-    self.host_log.write("Copying repo file to 'tmp' folder...")
-    params = self.shared_state
-    scp = SCP(params.user, params.sshkey_file, self.host, fileToCopy,
-              target, params.bootdir, self.host_log)
-    retcode1 = scp.run()
-    self.host_log.write("\n")
-
-    # Move file to repo dir
-    self.host_log.write("==========================\n")
-    self.host_log.write("Moving file to repo dir...")
-    targetDir = self.getRepoDir()
-    command = self.getMoveRepoFileCommand(targetDir)
-    ssh = SSH(params.user, params.sshkey_file, self.host, command,
-              params.bootdir, self.host_log)
-    retcode2 = ssh.run()
-    self.host_log.write("\n")
-    
-    # Change permissions on ambari.repo
-    self.host_log.write("==========================\n")
-    self.host_log.write("Changing permissions for ambari.repo...")
-    command = self.getRepoFileChmodCommand()
-    ssh = SSH(params.user, params.sshkey_file, self.host, command,
-              params.bootdir, self.host_log)
-    retcode4 = ssh.run()
-    self.host_log.write("\n")
+    if (os.path.exists(fileToCopy)):
+      self.host_log.write("==========================\n")
+      self.host_log.write("Copying repo file to 'tmp' folder...")
+      scp = SCP(params.user, params.sshkey_file, self.host, fileToCopy,
+                target, params.bootdir, self.host_log)
+      retcode1 = scp.run()
+      self.host_log.write("\n")
 
-    # Update repo cache for ubuntu OS
-    if OSCheck.is_ubuntu_family():
+      # Move file to repo dir
       self.host_log.write("==========================\n")
-      self.host_log.write("Update apt cache of repository...")
-      command = self.getAptUpdateCommand()
+      self.host_log.write("Moving file to repo dir...")
+      targetDir = self.getRepoDir()
+      command = self.getMoveRepoFileCommand(targetDir)
       ssh = SSH(params.user, params.sshkey_file, self.host, command,
                 params.bootdir, self.host_log)
       retcode2 = ssh.run()
       self.host_log.write("\n")
 
+      # Change permissions on ambari.repo
+      self.host_log.write("==========================\n")
+      self.host_log.write("Changing permissions for ambari.repo...")
+      command = self.getRepoFileChmodCommand()
+      ssh = SSH(params.user, params.sshkey_file, self.host, command,
+                params.bootdir, self.host_log)
+      retcode4 = ssh.run()
+      self.host_log.write("\n")
+
+      # Update repo cache for ubuntu OS
+      if OSCheck.is_ubuntu_family():
+        self.host_log.write("==========================\n")
+        self.host_log.write("Update apt cache of repository...")
+        command = self.getAptUpdateCommand()
+        ssh = SSH(params.user, params.sshkey_file, self.host, command,
+                  params.bootdir, self.host_log)
+        retcode2 = ssh.run()
+        self.host_log.write("\n")
+
+      retcode = max(retcode1["exitstatus"], retcode2["exitstatus"], retcode4["exitstatus"])
+    else:
+      self.host_log.write("==========================\n")
+      self.host_log.write("Copying required files...")
+      self.host_log.write("Ambari repo file not found: {0}".format(self.getRepoFile()))
+      retcode = -1
+      pass
+
+    # copy the setup script file
     self.host_log.write("==========================\n")
     self.host_log.write("Copying setup script file...")
     fileToCopy = params.setup_agent_file
@@ -534,7 +559,7 @@ class BootstrapDefault(Bootstrap):
     retcode3 = scp.run()
     self.host_log.write("\n")
 
-    return max(retcode1["exitstatus"], retcode2["exitstatus"], retcode3["exitstatus"], retcode4["exitstatus"])
+    return max(retcode, retcode3["exitstatus"])
 
   def getAmbariPort(self):
     server_port = self.shared_state.server_port

ambari-server/src/main/python/setupAgent.py

@@ -27,6 +27,7 @@ import subprocess
 
 from ambari_commons import OSCheck, OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from ambari_commons.os_utils import get_ambari_repo_file_full_name
 
 if OSCheck.is_windows_family():
   import urllib2
@@ -348,14 +349,29 @@ def run_setup(argv=None):
   if retcode["exitstatus"] == 0 and retcode["log"] != None and retcode["log"] != "" and retcode["log"][0].strip() != "":
     availableProjectVersion = retcode["log"].strip()
     if not isAgentPackageAlreadyInstalled(availableProjectVersion):
-      retcode = installAgent(availableProjectVersion)
-      if (not retcode["exitstatus"] == 0):
-        return retcode
-  elif retcode["exitstatus"] == 1 and retcode["log"][0].strip() != "":
-    return {"exitstatus": 1, "log": "Desired version ("+projectVersion+") of ambari-agent package"
+      # Verify that the ambari repo file is available before trying to install ambari-agent
+      ambari_repo_file = get_ambari_repo_file_full_name()
+      if os.path.exists(ambari_repo_file):
+        retcode = installAgent(availableProjectVersion)
+        if (not retcode["exitstatus"] == 0):
+          return retcode
+      else:
+        return {"exitstatus": 2, "log": "Ambari repo file not found: {0}".format(ambari_repo_file)}
+        pass
+  elif retcode["exitstatus"] == 1:
+    if retcode["log"] != None and retcode["log"] != "" and retcode["log"][0].strip() != "":
+      return {"exitstatus": 1, "log": "Desired version ("+projectVersion+") of ambari-agent package"
                                         " is not available."
                                         " Repository has following "
                                         "versions of ambari-agent:"+retcode["log"][0].strip()}
+    else:
+      # We are here because ambari-agent is not installed and version cannot be obtained from the repo file
+      logmessage = "Desired version ("+projectVersion+") of ambari-agent package is not available."
+      ambari_repo_file = get_ambari_repo_file_full_name()
+      if not os.path.exists(ambari_repo_file):
+        logmessage = logmessage + " " + "Ambari repo file not found: {0}".format(ambari_repo_file)
+      return {"exitstatus": retcode["exitstatus"], "log": logmessage}
+      pass
   else:
     return retcode
 

ambari-server/src/test/python/TestAmbariServer.py

@@ -1197,6 +1197,8 @@ class TestAmbariServer(TestCase):
     # Test ambari repo file permission change call
     #
 
+    # Test the case when ambari repo file is available
+
     # Reset the set_file_permissions() mock function
     set_file_permissions_mock.reset_mock()
 
@@ -1209,6 +1211,16 @@ class TestAmbariServer(TestCase):
     # Set up the mock function for os_utils.get_file_owner()
     get_file_owner_mock.return_value = "dummy.root"
 
+    # Set os.path.exists to return true when the input file is an ambari repo file
+    def file_exists_side_effect(*args, **kwargs):
+      if args[0] == get_ambari_repo_file_full_name_mock():
+        return True
+      else:
+        return False
+
+    exists_mock.side_effect = file_exists_side_effect
+    exists_mock.return_value = None
+
     try:
       # Clear the list of files whose permissions are to be changed
       configDefaults.NR_ADJUST_OWNERSHIP_LIST = [
@@ -1251,6 +1263,60 @@ class TestAmbariServer(TestCase):
       configDefaults.NR_ADJUST_OWNERSHIP_LIST = old_adjust_owner_list
     pass
 
+    #Test the case when ambari repo file is unavailable
+
+    # Reset the set_file_permissions() mock function
+    set_file_permissions_mock.reset_mock()
+
+    # Save the existing permissions list
+    old_adjust_owner_list = configDefaults.NR_ADJUST_OWNERSHIP_LIST
+
+    # Set up the mock function for os_utils.get_ambari_repo_file_full_name()
+    get_ambari_repo_file_full_name_mock.return_value = "ambari.dummy.repo"
+
+    # Set up the mock function for os_utils.get_file_owner()
+    get_file_owner_mock.return_value = "dummy.root"
+
+    # Set os.path.exists to return false always
+    exists_mock.side_effect = None
+    exists_mock.return_value = False
+
+    try:
+      # Clear the list of files whose permissions are to be changed
+      configDefaults.NR_ADJUST_OWNERSHIP_LIST = [
+      ]
+
+      # Call the function to be tested.
+      adjust_directory_permissions("dummy_user")
+
+      # One of the entries in NR_ADJUST_OWNERSHIP_LIST should be the full path to the ambari repo file.
+      # These are the expected values:
+
+      ambari_repo_file_entry = (
+        get_ambari_repo_file_full_name_mock(),
+        '644',
+        get_file_owner_mock(),
+        False
+      )
+
+      # Assert the arguments to the call set_file_permissions() - got from NR_ADJUST_OWNERSHIP_LIST
+      # Flag to ensure we found our entry in the set_file_permissions() call
+      entry_found = False
+
+      for args_entry in set_file_permissions_mock.call_args_list:
+        if args_entry[0][0] == ambari_repo_file_entry[0]:  # File name
+          entry_found = True
+          break
+
+      # Ensure that the ambari repo file entry was not found
+      self.assertFalse(entry_found)
+
+    finally:
+      # Restore the permissions list
+      configDefaults.NR_ADJUST_OWNERSHIP_LIST = old_adjust_owner_list
+    pass
+
+
   @not_for_platform(PLATFORM_WINDOWS)
   @patch("os.path.exists")
   @patch("ambari_commons.os_linux.os_run_os_command")

ambari-server/src/test/python/TestBootstrap.py

@@ -407,7 +407,7 @@ class TestBootstrap(TestCase):
     rf = bootstrap_obj.getMoveRepoFileCommand("target")
     self.assertEquals(rf, "sudo -S mv RemoteName target/ambari.repo < RemoteName")
 
-
+  @patch("os.path.exists")
   @patch.object(OSCheck, "is_suse_family")
   @patch.object(OSCheck, "is_ubuntu_family")
   @patch.object(OSCheck, "is_redhat_family")
@@ -423,7 +423,20 @@ class TestBootstrap(TestCase):
   def test_copyNeededFiles(self, write_mock, ssh_run_mock, ssh_init_mock,
                            scp_run_mock, scp_init_mock,
                            getRemoteName_mock, getRepoFile_mock, getRepoDir,
-                           getMoveRepoFileCommand, is_redhat_family, is_ubuntu_family, is_suse_family):
+                           getMoveRepoFileCommand, is_redhat_family, is_ubuntu_family, is_suse_family,
+                           os_path_exists_mock):
+    #
+    # Ambari repo file exists
+    #
+    def os_path_exists_side_effect(*args, **kwargs):
+      if args[0] == getRepoFile_mock():
+        return True
+      else:
+        return False
+
+    os_path_exists_mock.side_effect = os_path_exists_side_effect
+    os_path_exists_mock.return_value = None
+
     shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
                                "setupAgentFile", "ambariServer", "centos6",
                                None, "8440", "root")
@@ -471,6 +484,22 @@ class TestBootstrap(TestCase):
     res = bootstrap_obj.copyNeededFiles()
     self.assertEquals(res, expected3["exitstatus"])
 
+    #
+    #Ambari repo file does not exist
+    #
+    os_path_exists_mock.side_effect = None
+    os_path_exists_mock.return_value = False
+
+    #Expectations:
+    # SSH will not be called at all
+    # SCP will be called once for copying the setup script file
+    scp_run_mock.reset_mock()
+    ssh_run_mock.reset_mock()
+    expectedResult = {"exitstatus": 33, "log": "log33", "errormsg": "errorMsg"}
+    scp_run_mock.side_effect = [expectedResult]
+    res = bootstrap_obj.copyNeededFiles()
+    self.assertFalse(ssh_run_mock.called)
+    self.assertEquals(res, expectedResult["exitstatus"])
 
   @patch.object(BootstrapDefault, "getOsCheckScriptRemoteLocation")
   @patch.object(SSH, "__init__")

ambari-server/src/test/python/TestSetupAgent.py

@@ -326,6 +326,8 @@ class TestSetupAgent(TestCase):
     self.assertEqual(retval["exitstatus"], 0)
     pass
 
+  @patch("os.path.exists")
+  @patch.object(setup_agent, "get_ambari_repo_file_full_name")
   @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
   @patch.object(setup_agent, 'checkVerbose')
   @patch.object(setup_agent, 'isAgentPackageAlreadyInstalled')
@@ -342,7 +344,8 @@ class TestSetupAgent(TestCase):
   def test_setup_agent_main(self, dirname_mock, realpath_mock, exit_mock, checkServerReachability_mock,
                             getOptimalVersion_mock, is_ubuntu_family_mock, is_suse_family_mock,
                             installAgent_mock, configureAgent_mock, runAgent_mock,
-                            isAgentPackageAlreadyInstalled_mock, checkVerbose_mock):
+                            isAgentPackageAlreadyInstalled_mock, checkVerbose_mock,
+                            get_ambari_repo_file_full_name_mock, os_path_exists_mock):
     checkServerReachability_mock.return_value = {'log': 'log', 'exitstatus': 0}
     installAgent_mock.return_value = {'log': 'log', 'exitstatus': 0}
     configureAgent_mock.return_value = {'log': 'log', 'exitstatus': 0}
@@ -355,6 +358,7 @@ class TestSetupAgent(TestCase):
     self.assertEqual(ret["exitstatus"], 1)
     self.assertTrue(getOptimalVersion_mock.called)
 
+    # A version of ambari-agent was installed or available in the repo file
     exit_mock.reset_mock()
     getOptimalVersion_mock.reset_mock()
 
@@ -362,6 +366,16 @@ class TestSetupAgent(TestCase):
     isAgentPackageAlreadyInstalled_mock.return_value = False
     is_suse_family_mock.return_value = True
     is_ubuntu_family_mock.return_value = False
+    get_ambari_repo_file_full_name_mock.return_value = "Ambari_Repo_File"
+
+    def os_path_exists_side_effect(*args, **kwargs):
+      # Ambari repo file exists
+      if (args[0] == get_ambari_repo_file_full_name_mock()):
+        return True
+      else:
+        return False
+
+    os_path_exists_mock.side_effect = os_path_exists_side_effect
     ret = setup_agent.main(("setupAgent.py","agents_host","password", "server_hostname","1.1.1","8080"))
     self.assertFalse(exit_mock.called)
     self.assertTrue(getOptimalVersion_mock.called)
@@ -471,6 +485,50 @@ class TestSetupAgent(TestCase):
     self.assertFalse(exit_mock.called)
     self.assertTrue("exitstatus" in ret)
     self.assertEqual(ret["exitstatus"], 1)
+
+    # A version of ambari-agent was installed and ambari repo file was not found
+    exit_mock.reset_mock()
+    installAgent_mock.reset_mock()
+    getOptimalVersion_mock.reset_mock()
+    os_path_exists_mock.side_effect = None
+    os_path_exists_mock.return_value = False #ambari repo file was not found
+    getOptimalVersion_mock.return_value = {'log': '1.1.1', 'exitstatus': 0}
+    isAgentPackageAlreadyInstalled_mock.return_value = False
+    is_suse_family_mock.return_value = True
+    is_ubuntu_family_mock.return_value = False
+    get_ambari_repo_file_full_name_mock.return_value = "Ambari_Repo_File"
+
+    ret = setup_agent.main(("setupAgent.py","agents_host","password", "server_hostname","1.1.1","8080"))
+    self.assertFalse(exit_mock.called)
+    self.assertTrue(getOptimalVersion_mock.called)
+    self.assertTrue(isAgentPackageAlreadyInstalled_mock.called)
+    self.assertFalse(installAgent_mock.called) # ambari repo file is missing, so installAgent() will not be called
+    self.assertFalse(is_suse_family_mock.called)
+    self.assertFalse(is_ubuntu_family_mock.called)
+    self.assertTrue("exitstatus" in ret)
+    self.assertEqual(ret["exitstatus"], 2) # ambari repo file not found error code
+    self.assertTrue(get_ambari_repo_file_full_name_mock() in ret["log"]) # ambari repo file not found message
+
+    # ambari-agent was not installed and ambari repo file was not found
+    exit_mock.reset_mock()
+    os_path_exists_mock.side_effect = None
+    os_path_exists_mock.return_value = False # ambari repo file not found
+    getOptimalVersion_mock.reset_mock()
+    isAgentPackageAlreadyInstalled_mock.reset_mock()
+    is_suse_family_mock.reset_mock()
+    is_ubuntu_family_mock.reset_mock()
+    installAgent_mock.reset_mock()
+
+    getOptimalVersion_mock.return_value = {'log': '', 'exitstatus': 1}     #ambari-agent not installed
+    ret = setup_agent.main(("setupAgent.py","agents_host","password", "server_hostname","1.1.1","8080"))
+    self.assertFalse(exit_mock.called)
+    self.assertTrue(getOptimalVersion_mock.called)
+    self.assertFalse(isAgentPackageAlreadyInstalled_mock.called)
+    self.assertFalse(is_suse_family_mock.called)
+    self.assertFalse(is_ubuntu_family_mock.called)
+    self.assertTrue("exitstatus" in ret)
+    self.assertEqual(ret["exitstatus"], 1)
+    self.assertTrue(get_ambari_repo_file_full_name_mock() in ret["log"]) # ambari repo file not found message
     pass
 
   @patch.object(setup_agent, 'execOsCommand')