AMBARI-9677 - Ranger Prevents HDFS NameNode From Restarting Due To Malformed File Location (jonathanhurley)

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py

@@ -50,14 +50,16 @@ def setup_ranger_hbase():
     if return_code != 0:
       raise Fail('Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
 
-    hdp_version = re.sub('hbase-client - ', '', hdp_output)
+    hdp_version = re.sub('hbase-client - ', '', hdp_output).strip()
     match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
 
     if match is None:
       raise Fail('Failed to get extracted version')
 
     file_path = '/usr/hdp/'+ hdp_version +'/ranger-hbase-plugin/install.properties'
-
+    if not os.path.isfile(file_path):
+      raise Fail('Ranger HBase plugin install.properties file does not exist at {0}'.format(file_path))
+    
     ranger_hbase_dict = ranger_hbase_properties()
     hbase_repo_data = hbase_repo_properties()
 

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py

@@ -52,18 +52,21 @@ def setup_ranger_hdfs():
       raise Fail(
         'Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
 
-    hdp_version = re.sub('hadoop-client - ', '', hdp_output)
+    hdp_version = re.sub('hadoop-client - ', '', hdp_output).strip()
     match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
 
     if match is None:
       raise Fail('Failed to get extracted version')
 
     file_path = '/usr/hdp/' + hdp_version + '/ranger-hdfs-plugin/install.properties'
+    if not os.path.isfile(file_path):
+      raise Fail('Ranger HDFS plugin install.properties file does not exist at {0}'.format(file_path))
 
     ranger_hdfs_dict = ranger_hdfs_properties()
     hdfs_repo_data = hdfs_repo_properties()
 
-    write_properties_to_file(file_path, ranger_hdfs_dict)
+    if os.path.isfile(file_path):
+      write_properties_to_file(file_path, ranger_hdfs_dict)
 
     if params.enable_ranger_hdfs:
       cmd = format('cd /usr/hdp/{hdp_version}/ranger-hdfs-plugin/ && sh enable-hdfs-plugin.sh')

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py

@@ -50,19 +50,21 @@ def setup_ranger_hive():
     if return_code != 0:
       raise Fail('Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
 
-    hdp_version = re.sub('hive-server2 - ', '', hdp_output)
+    hdp_version = re.sub('hive-server2 - ', '', hdp_output).strip()
     match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
 
     if match is None:
       raise Fail('Failed to get extracted version')
 
     file_path = '/usr/hdp/'+ hdp_version +'/ranger-hive-plugin/install.properties'
+    if not os.path.isfile(file_path):
+      raise Fail('Ranger Hive plugin install.properties file does not exist at {0}'.format(file_path))
 
     ranger_hive_dict = ranger_hive_properties()
     hive_repo_data = hive_repo_properties()
 
     write_properties_to_file(file_path, ranger_hive_dict)
-  
+
     if params.enable_ranger_hive:
       cmd = format('cd /usr/hdp/{hdp_version}/ranger-hive-plugin/ && sh enable-hive-plugin.sh')
       ranger_adm_obj = Rangeradmin(url=ranger_hive_dict['POLICY_MGR_URL'])

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py

@@ -50,16 +50,18 @@ def setup_ranger_knox():
     if return_code != 0:
       raise Fail('Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
 
-    hdp_version = re.sub('knox-server - ', '', hdp_output)
+    hdp_version = re.sub('knox-server - ', '', hdp_output).strip()
     match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
 
     if match is None:
       raise Fail('Failed to get extracted version')
 
     file_path = '/usr/hdp/'+ hdp_version +'/ranger-knox-plugin/install.properties'
+    if not os.path.isfile(file_path):
+      raise Fail('Ranger Knox plugin install.properties file does not exist at {0}'.format(file_path))
 
     ranger_knox_dict = ranger_knox_properties()
-    knox_repo_data = knox_repo_properties()     
+    knox_repo_data = knox_repo_properties()
 
     write_properties_to_file(file_path, ranger_knox_dict)
 

ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-env.xml

@@ -50,27 +50,27 @@
     <name>ranger_admin_username</name>
     <value>amb_ranger_admin</value>
     <property-type>TEXT</property-type>
-    <description></description>
+    <description>This is the ambari user created for creating repositories and policies in Ranger Admin for each plugin</description>
   </property>
 
   <property>
     <name>ranger_admin_password</name>
     <value>ambari123</value>
     <property-type>PASSWORD</property-type>
-    <description></description>
+    <description>This is the ambari user password created for creating repositories and policies in Ranger Admin for each plugin</description>
   </property>
 
   <property>
     <name>admin_username</name>
     <value>admin</value>
-    <description>This user must be synced with Ranger admin</description>
+    <description>This is the username for default admin user that is used for creating ambari user in Ranger Admin</description>
   </property>  
 
   <property>
     <name>admin_password</name>
     <value>admin</value>
     <property-type>PASSWORD</property-type>
-    <description>This admin user password must be synced with Ranger admin</description>
+    <description>This is the password for default admin user that is used for creating ambari user in Ranger Admin</description>
   </property>
   
   <property>

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py

@@ -29,6 +29,7 @@ hdp_stack_version         = format_hdp_stack_version(hdp_stack_version)
 stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 version = default("/commandParams/version", None)
+stack_name = default("/hostLevelParams/stack_name", None)
 
 if stack_is_hdp22_or_further:
   ranger_home    = '/usr/hdp/current/ranger-admin'

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py

@@ -89,7 +89,7 @@ def do_post_installation():
   ranger_site['http.enabled'] = params.http_enabled
   write_properties_to_file(file_path, ranger_site)
   Logger.info('Performing Ranger post installation DONE')
-  Execute(format('chmod 744 {params.ranger_conf}/ranger_webserver.properties'))
+  File(format('{params.ranger_conf}/ranger_webserver.properties'), mode=0744)
 
 def setup_usersync():
   import params

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py

@@ -51,15 +51,15 @@ def setup_ranger_storm():
     if return_code != 0:
       raise Fail('Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
 
-    cmd_split = hdp_output.strip().split( ) 
-    hdp_version = cmd_split[2]
-
+    hdp_version = re.sub('storm-nimbus - ', '', hdp_output).strip()
     match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
 
     if match is None:
       raise Fail('Failed to get extracted version')
 
     file_path = '/usr/hdp/'+ hdp_version +'/ranger-storm-plugin/install.properties'
+    if not os.path.isfile(file_path):
+      raise Fail('Ranger Storm plugin install.properties file does not exist at {0}'.format(file_path))
 
     ranger_storm_dict = ranger_storm_properties()
     storm_repo_data = storm_repo_properties()        

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py

@@ -19,12 +19,11 @@ limitations under the License.
 from ambari_commons import OSCheck
 '''
 from stacks.utils.RMFTestCase import *
-from ambari_commons import OSCheck
 from mock.mock import MagicMock, patch
 import resource_management
 from resource_management.core import shell
+from resource_management.core.exceptions import Fail
 
-import subprocess
 
 @patch.object(shell, "call", new=MagicMock(return_value=(1,"")))
 class TestNamenode(RMFTestCase):
@@ -780,6 +779,27 @@ class TestNamenode(RMFTestCase):
       )
       self.assertNoMoreResources()
 
+  @patch("os.path.isfile")
+  def test_ranger_installed_missing_file(self, isfile_mock):
+    """
+    Tests that when Ranger is enabled for HDFS, that an exception is thrown
+    if there is no install.properties found
+    :return:
+    """
+    isfile_mock.return_value = False
+
+    try:
+      self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+        classname = "NameNode", command = "start", config_file = "ranger-namenode-start.json",
+        hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES )
+
+      self.fail("Expected a failure since the ranger install.properties was missing")
+    except Fail, failure:
+      pass
+
+    self.assertTrue(isfile_mock.called)
+
+
 class Popen_Mock:
   return_value = 1
   lines = ['Time Stamp               Iteration#  Bytes Already Moved  Bytes Left To Move  Bytes Being Moved\n',