AMBARI-5611. Datanode and other components fail to start without YARN installed (aonishuk)

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py

@@ -105,9 +105,6 @@ if rca_enabled == True:
 else:
   rca_prefix = rca_disabled_prefix
 
-#mapred-env.sh
-jobhistory_heapsize = config['configurations']['global']['jobhistory_heapsize']
-
 #hadoop-env.sh
 java_home = config['hostLevelParams']['java_home']
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py

@@ -60,7 +60,7 @@ def setup_hadoop():
   else:
     tc_owner = params.hdfs_user
 
-  for file in ['hadoop-env.sh', 'commons-logging.properties', 'mapred-env.sh']:
+  for file in ['hadoop-env.sh', 'commons-logging.properties']:
     File(os.path.join(params.hadoop_conf_dir, file),
          owner=tc_owner,
          content=Template(file + ".j2")
@@ -194,4 +194,4 @@ def init_services():
     )
       
   
-  
+  

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py

@@ -105,6 +105,7 @@ yarn_log_aggregation_enabled = config['configurations']['yarn-site']['yarn.log-a
 yarn_nm_app_log_dir =  config['configurations']['yarn-site']['yarn.nodemanager.remote-app-log-dir']
 mapreduce_jobhistory_intermediate_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.intermediate-done-dir']
 mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.done-dir']
+jobhistory_heapsize = config['configurations']['global']['jobhistory_heapsize']
 
 #for create_hdfs_directory
 hostname = config["hostname"]

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py

@@ -159,6 +159,11 @@ def yarn(name = None):
     tc_mode = None
     tc_owner = params.hdfs_user
 
+  File(format("{config_dir}/mapred-env.sh"),
+       owner=tc_owner,
+       content=Template('mapred-env.sh.j2')
+  )
+
   if params.security_enabled:
     File(os.path.join(params.hadoop_bin, "task-controller"),
          owner="root",

ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py

@@ -266,6 +266,10 @@ class TestHistoryServer(RMFTestCase):
       group = 'hadoop',
       mode = 0755,
     )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'hdfs',
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg',
                               content = Template('taskcontroller.cfg.j2'),
                               owner = 'hdfs',
@@ -450,6 +454,10 @@ class TestHistoryServer(RMFTestCase):
       group = 'hadoop',
       mode = 0644,
     )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'root',
+                              )
     self.assertResourceCalled('File', '/usr/lib/hadoop/sbin/task-controller',
                               owner = 'root',
                               group = 'hadoop',
@@ -488,4 +496,4 @@ class TestHistoryServer(RMFTestCase):
     self.assertResourceCalled('File', '/etc/hadoop/conf/ssl-server.xml.example',
                               owner = 'mapred',
                               group = 'hadoop',
-                              )
+                              )

ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py

@@ -125,6 +125,10 @@ class TestMapReduce2Client(RMFTestCase):
       group = 'hadoop',
       mode = 0755,
     )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'hdfs',
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg',
                               content = Template('taskcontroller.cfg.j2'),
                               owner = 'hdfs',
@@ -256,6 +260,10 @@ class TestMapReduce2Client(RMFTestCase):
       group = 'hadoop',
       mode = 0644,
     )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'root',
+                              )
     self.assertResourceCalled('File', '/usr/lib/hadoop/sbin/task-controller',
                               owner = 'root',
                               group = 'hadoop',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py

@@ -266,6 +266,10 @@ class TestNodeManager(RMFTestCase):
       group = 'hadoop',
       mode = 0755,
     )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'hdfs',
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg',
                               content = Template('taskcontroller.cfg.j2'),
                               owner = 'hdfs',
@@ -450,6 +454,10 @@ class TestNodeManager(RMFTestCase):
       group = 'hadoop',
       mode = 0644,
     )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'root',
+                              )
     self.assertResourceCalled('File', '/usr/lib/hadoop/sbin/task-controller',
                               owner = 'root',
                               group = 'hadoop',
@@ -488,4 +496,4 @@ class TestNodeManager(RMFTestCase):
     self.assertResourceCalled('File', '/etc/hadoop/conf/ssl-server.xml.example',
                               owner = 'mapred',
                               group = 'hadoop',
-                              )
+                              )

ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py

@@ -210,6 +210,10 @@ class TestResourceManager(RMFTestCase):
       group = 'hadoop',
       mode = 0755,
     )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'hdfs',
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg',
                               content = Template('taskcontroller.cfg.j2'),
                               owner = 'hdfs',
@@ -339,6 +343,10 @@ class TestResourceManager(RMFTestCase):
       group = 'hadoop',
       mode = 0644,
     )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'root',
+                              )
     self.assertResourceCalled('File', '/usr/lib/hadoop/sbin/task-controller',
                               owner = 'root',
                               group = 'hadoop',
@@ -377,4 +385,4 @@ class TestResourceManager(RMFTestCase):
     self.assertResourceCalled('File', '/etc/hadoop/conf/ssl-server.xml.example',
                               owner = 'mapred',
                               group = 'hadoop',
-                              )
+                              )

ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py

@@ -126,6 +126,10 @@ class TestYarnClient(RMFTestCase):
       group = 'hadoop',
       mode = 0755,
     )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'hdfs',
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg',
                               content = Template('taskcontroller.cfg.j2'),
                               owner = 'hdfs',
@@ -257,6 +261,10 @@ class TestYarnClient(RMFTestCase):
       group = 'hadoop',
       mode = 0644,
     )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'root',
+                              )
     self.assertResourceCalled('File', '/usr/lib/hadoop/sbin/task-controller',
                               owner = 'root',
                               group = 'hadoop',
@@ -397,6 +405,10 @@ class TestYarnClient(RMFTestCase):
       group = 'hadoop',
       mode = 0755,
     )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'hdfs',
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg',
                               content = Template('taskcontroller.cfg.j2'),
                               owner = 'hdfs',

ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py

@@ -63,10 +63,6 @@ class TestHookBeforeStart(RMFTestCase):
                               content = Template('commons-logging.properties.j2'),
                               owner = 'hdfs',
                               )
-    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
-                              content = Template('mapred-env.sh.j2'),
-                              owner = 'hdfs',
-                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/health_check',
                               content = Template('health_check-v2.j2'),
                               owner = 'hdfs',
@@ -149,10 +145,6 @@ class TestHookBeforeStart(RMFTestCase):
                               content = Template('commons-logging.properties.j2'),
                               owner = 'root',
                               )
-    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
-                              content = Template('mapred-env.sh.j2'),
-                              owner = 'root',
-                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/health_check',
                               content = Template('health_check-v2.j2'),
                               owner = 'root',

ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py

@@ -160,6 +160,10 @@ class TestAppTimelineServer(RMFTestCase):
                               group = 'hadoop',
                               mode = 0755,
                               )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'hdfs',
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg',
                               content = Template('taskcontroller.cfg.j2'),
                               owner = 'hdfs',