AMBARI-8076. Install on a 5 node cluster fails with link creation for libsnappy. (aonishuk)

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py

@@ -33,6 +33,7 @@ user_group = config['configurations']['cluster-env']['user_group']
 
 #hosts
 hostname = config["hostname"]
+current_service = config['serviceName']
 ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
 rm_host = default("/clusterHostInfo/rm_host", [])
 slave_hosts = default("/clusterHostInfo/slave_hosts", [])

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/shared_initialization.py

@@ -31,7 +31,8 @@ def setup_hadoop():
           only_if="test -f /selinux/enforce"
   )
 
-  install_snappy()
+  if params.current_service == "HDFS":
+    install_snappy()
 
 
   if params.has_namenode:

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py

@@ -42,6 +42,7 @@ else:
   hadoop_bin = "/usr/lib/hadoop/sbin"
   hadoop_home = '/usr'
 
+current_service = config['serviceName']
 hadoop_conf_dir = "/etc/hadoop/conf"
 #security params
 security_enabled = config['configurations']['cluster-env']['security_enabled']

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py

@@ -31,7 +31,8 @@ def setup_hadoop():
           only_if="test -f /selinux/enforce"
   )
 
-  install_snappy()
+  if params.current_service == "HDFS":
+    install_snappy()
 
   #directories
   if params.has_namenode:

ambari-server/src/test/python/stacks/1.3.2/hooks/before-START/test_before_start.py

@@ -21,6 +21,7 @@ limitations under the License.
 from mock.mock import MagicMock, call, patch
 from resource_management import *
 from stacks.utils.RMFTestCase import *
+import json
 
 @patch("os.path.exists", new = MagicMock(return_value=True))
 @patch.object(Hook, "run_custom_hook")
@@ -35,10 +36,6 @@ class TestHookBeforeStart(RMFTestCase):
     self.assertResourceCalled('Execute', '/bin/echo 0 > /selinux/enforce',
                               only_if = 'test -f /selinux/enforce',
                               )
-    self.assertResourceCalled('Execute', 'mkdir -p /usr/lib/hadoop/lib/native/Linux-i386-32; ln -sf /usr/lib/libsnappy.so /usr/lib/hadoop/lib/native/Linux-i386-32/libsnappy.so',
-                              )
-    self.assertResourceCalled('Execute', 'mkdir -p /usr/lib/hadoop/lib/native/Linux-amd64-64; ln -sf /usr/lib64/libsnappy.so /usr/lib/hadoop/lib/native/Linux-amd64-64/libsnappy.so',
-                              )
     self.assertResourceCalled('Directory', '/var/log/hadoop',
                               owner = 'root',
                               group = 'hadoop',
@@ -91,6 +88,62 @@ class TestHookBeforeStart(RMFTestCase):
                        command="hook",
                        config_file="secured.json"
     )
+    self.assertResourceCalled('Execute', '/bin/echo 0 > /selinux/enforce',
+                              only_if = 'test -f /selinux/enforce',
+                              )
+    self.assertResourceCalled('Directory', '/var/log/hadoop',
+                              owner = 'root',
+                              group = 'hadoop',
+                              mode = 0775,
+                              recursive = True,
+                              )
+    self.assertResourceCalled('Directory', '/var/run/hadoop',
+                              owner = 'root',
+                              group = 'root',
+                              recursive = True,
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/health_check',
+                              content = Template('health_check.j2'),
+                              owner = 'root',
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/log4j.properties',
+                              content = 'log4jproperties\nline2',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              mode = 0644,
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/hadoop-metrics2.properties',
+                              content = Template('hadoop-metrics2.properties.j2'),
+                              owner = 'hdfs',
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/task-log4j.properties',
+                              content = StaticFile('task-log4j.properties'),
+                              mode = 0755,
+                              )
+    self.assertResourceCalled('Link', '/usr/lib/hadoop/lib/hadoop-tools.jar',
+                              to = '/usr/lib/hadoop/hadoop-tools.jar',
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/configuration.xsl',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/masters',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              )
+    self.assertNoMoreResources()
+
+  def test_hook_default_hdfs(self, mockHook):
+    config_file = "stacks/1.3.2/configs/default.json"
+    with open(config_file, "r") as f:
+      default_json = json.load(f)
+
+    default_json['serviceName']= 'HDFS'
+    self.executeScript("1.3.2/hooks/before-START/scripts/hook.py",
+                       classname="BeforeStartHook",
+                       command="hook",
+                       config_dict=default_json
+    )
     self.assertResourceCalled('Execute', '/bin/echo 0 > /selinux/enforce',
                               only_if = 'test -f /selinux/enforce',
                               )
@@ -109,12 +162,16 @@ class TestHookBeforeStart(RMFTestCase):
                               group = 'root',
                               recursive = True,
                               )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/commons-logging.properties',
+                              content = Template('commons-logging.properties.j2'),
+                              owner = 'hdfs',
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/health_check',
                               content = Template('health_check.j2'),
-                              owner = 'root',
+                              owner = 'hdfs',
                               )
     self.assertResourceCalled('File', '/etc/hadoop/conf/log4j.properties',
-                              content = 'log4jproperties\nline2',
+                              content = 'log4jproperties\nline2log4jproperties\nline2\nambari.jobhistory.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nambari.jobhistory.driver=org.postgresql.Driver\nambari.jobhistory.user=mapred\nambari.jobhistory.password=mapred\nambari.jobhistory.logger=${hadoop.root.logger}\n\nlog4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender\nlog4j.appender.JHA.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nlog4j.appender.JHA.driver=org.postgresql.Driver\nlog4j.appender.JHA.user=mapred\nlog4j.appender.JHA.password=mapred\n\nlog4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA\nlog4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true\n\n',
                               owner = 'hdfs',
                               group = 'hadoop',
                               mode = 0644,

ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py

@@ -21,6 +21,7 @@ limitations under the License.
 from mock.mock import MagicMock, call, patch
 from resource_management import *
 from stacks.utils.RMFTestCase import *
+import json
 
 @patch("os.path.exists", new = MagicMock(return_value=True))
 @patch.object(Hook, "run_custom_hook", new = MagicMock())
@@ -34,10 +35,6 @@ class TestHookBeforeStart(RMFTestCase):
     self.assertResourceCalled('Execute', '/bin/echo 0 > /selinux/enforce',
                               only_if = 'test -f /selinux/enforce',
                               )
-    self.assertResourceCalled('Execute', 'mkdir -p /usr/lib/hadoop/lib/native/Linux-i386-32; ln -sf /usr/lib/libsnappy.so /usr/lib/hadoop/lib/native/Linux-i386-32/libsnappy.so',
-                              )
-    self.assertResourceCalled('Execute', 'mkdir -p /usr/lib/hadoop/lib/native/Linux-amd64-64; ln -sf /usr/lib64/libsnappy.so /usr/lib/hadoop/lib/native/Linux-amd64-64/libsnappy.so',
-                              )
     self.assertResourceCalled('Directory', '/var/log/hadoop',
                               owner = 'root',
                               group = 'hadoop',
@@ -92,6 +89,68 @@ class TestHookBeforeStart(RMFTestCase):
                        command="hook",
                        config_file="secured.json"
     )
+    self.assertResourceCalled('Execute', '/bin/echo 0 > /selinux/enforce',
+                              only_if = 'test -f /selinux/enforce',
+                              )
+    self.assertResourceCalled('Directory', '/var/log/hadoop',
+                              owner = 'root',
+                              group = 'hadoop',
+                              mode = 0775,
+                              recursive = True,
+                              )
+    self.assertResourceCalled('Directory', '/var/run/hadoop',
+                              owner = 'root',
+                              group = 'root',
+                              recursive = True,
+                              )
+    self.assertResourceCalled('Directory', '/tmp/hadoop-hdfs',
+                              owner = 'hdfs',
+                              recursive = True,
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/commons-logging.properties',
+                              content = Template('commons-logging.properties.j2'),
+                              owner = 'root',
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/health_check',
+                              content = Template('health_check-v2.j2'),
+                              owner = 'root',
+                              )
+    self.assertResourceCalled('File',
+                              '/etc/hadoop/conf/log4j.properties',
+                              mode=0644,
+                              group='hadoop',
+                              owner='hdfs',
+                              content='log4jproperties\nline2log4jproperties\nline2'
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/hadoop-metrics2.properties',
+                              content = Template('hadoop-metrics2.properties.j2'),
+                              owner = 'hdfs',
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/task-log4j.properties',
+                              content = StaticFile('task-log4j.properties'),
+                              mode = 0755,
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/configuration.xsl',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/masters',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              )
+    self.assertNoMoreResources()
+
+  def test_hook_default_hdfs(self):
+    config_file = "stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      default_json = json.load(f)
+
+    default_json['serviceName']= 'HDFS'
+    self.executeScript("2.0.6/hooks/before-START/scripts/hook.py",
+                       classname="BeforeStartHook",
+                       command="hook",
+                       config_dict=default_json
+    )
     self.assertResourceCalled('Execute', '/bin/echo 0 > /selinux/enforce',
                               only_if = 'test -f /selinux/enforce',
                               )
@@ -116,11 +175,11 @@ class TestHookBeforeStart(RMFTestCase):
                               )
     self.assertResourceCalled('File', '/etc/hadoop/conf/commons-logging.properties',
                               content = Template('commons-logging.properties.j2'),
-                              owner = 'root',
+                              owner = 'hdfs',
                               )
     self.assertResourceCalled('File', '/etc/hadoop/conf/health_check',
                               content = Template('health_check-v2.j2'),
-                              owner = 'root',
+                              owner = 'hdfs',
                               )
     self.assertResourceCalled('File',
                               '/etc/hadoop/conf/log4j.properties',
@@ -128,7 +187,7 @@ class TestHookBeforeStart(RMFTestCase):
                               group='hadoop',
                               owner='hdfs',
                               content='log4jproperties\nline2log4jproperties\nline2'
-                              )
+    )
     self.assertResourceCalled('File', '/etc/hadoop/conf/hadoop-metrics2.properties',
                               content = Template('hadoop-metrics2.properties.j2'),
                               owner = 'hdfs',
@@ -146,7 +205,7 @@ class TestHookBeforeStart(RMFTestCase):
                               group = 'hadoop',
                               )
     self.assertNoMoreResources()
-    
+
 def test_that_jce_is_required_in_secured_cluster(self):
   try:
     self.executeScript("2.0.6/hooks/before-START/scripts/hook.py",