AMBARI-4222. Add historyserver to HDP1 on backend (aonishuk)

ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/hdfs_namenode.py

@@ -101,23 +101,16 @@ def create_app_directories():
                    mode="755"
     )
   if len(params.hs_host) != 0:
-    if params.yarn_log_aggregation_enabled:
-      hdfs_directory(name=params.yarn_nm_app_log_dir,
-                     owner=params.yarn_user,
-                     group=params.user_group,
-                     mode="1777",
-                     recursive_chmod=True
-      )
     hdfs_directory(name=params.mapreduce_jobhistory_intermediate_done_dir,
                    owner=params.mapred_user,
                    group=params.user_group,
-                   mode="1777"
+                   mode="777"
     )
 
     hdfs_directory(name=params.mapreduce_jobhistory_done_dir,
                    owner=params.mapred_user,
                    group=params.user_group,
-                   mode="1777"
+                   mode="777"
     )
 
   pass

ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/params.py

@@ -127,8 +127,6 @@ hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']#",
 hbase_staging_dir = "/apps/hbase/staging"
 hive_apps_whs_dir = config['configurations']['hive-site']["hive.metastore.warehouse.dir"] #, "/apps/hive/warehouse")
 webhcat_apps_dir = "/apps/webhcat"
-yarn_log_aggregation_enabled = config['configurations']['yarn-site']['yarn.log-aggregation-enable']#","true")
-yarn_nm_app_log_dir =  config['configurations']['yarn-site']['yarn.nodemanager.remote-app-log-dir']#","/app-logs")
 mapreduce_jobhistory_intermediate_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.intermediate-done-dir']#","/app-logs")
 mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.done-dir']#","/mr-history/done")
 

ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/configuration/mapred-site.xml

@@ -554,6 +554,28 @@
     <description>The keytab for the job history server principal.</description>
   </property>
 
+  <property>
+    <name>mapreduce.jobhistory.intermediate-done-dir</name>
+    <value>/mr-history/tmp</value>
+    <description>
+      Directory where history files are written by MapReduce jobs.
+    </description>
+  </property>
+
+  <property>
+    <name>mapreduce.jobhistory.done-dir</name>
+    <value>/mr-history/done</value>
+    <description>
+      Directory where history files are managed by the MR JobHistory Server.
+    </description>
+  </property>
+
+  <property>       
+    <name>mapreduce.jobhistory.webapp.address</name>
+    <value>localhost:19888</value>
+    <description>Enter your JobHistoryServer hostname.</description>
+  </property>
+
   <property>
     <name>mapred.jobtracker.blacklist.fault-timeout-window</name>
     <value>180</value>

ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/metainfo.xml

@@ -54,6 +54,17 @@
             <timeout>600</timeout>
           </commandScript>
         </component>
+        
+              
+	    <component>
+	      <name>HISTORYSERVER</name>
+	      <category>MASTER</category>
+	      <commandScript>
+            <script>scripts/historyserver.py</script>
+            <scriptType>PYTHON</scriptType>
+            <timeout>600</timeout>
+          </commandScript>
+	    </component>
       </components>
 
 

ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/historyserver.py

@@ -26,14 +26,14 @@ from mapreduce import mapreduce
 from service import service
 
 class Historyserver(Script):
-  # def install(self, env):
-  #   self.install_packages(env)
-  #   self.configure(env)
-  #
-  # def configure(self, env):
-  #   import params
-  #   env.set_params(params)
-  #   mapreduce()
+  def install(self, env):
+    self.install_packages(env)
+    self.configure(env)
+  
+  def configure(self, env):
+    import params
+    env.set_params(params)
+    mapreduce()
 
   def start(self, env):
     import params
@@ -50,10 +50,10 @@ class Historyserver(Script):
             action='stop'
     )
 
-  # def status(self, env):
-  #   import status_params
-  #   env.set_params(status_params)
-  #   check_process_status(status_params.historyserver_pid_file)
+  def status(self, env):
+     import status_params
+     env.set_params(status_params)
+     check_process_status(status_params.historyserver_pid_file)
 
 if __name__ == "__main__":
   Historyserver().execute()

ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/jobtracker.py

@@ -46,7 +46,6 @@ from resource_management import *
 
 from mapreduce import mapreduce
 from service import service
-from historyserver import Historyserver
 
 class Jobtracker(Script):
   def install(self, env):
@@ -65,9 +64,7 @@ class Jobtracker(Script):
     service('jobtracker',
             action='start'
     )
-
-    Historyserver().start(env) #TODO HS will be separate component
-
+    
   def stop(self, env):
     import params
     env.set_params(params)
@@ -75,9 +72,7 @@ class Jobtracker(Script):
     service('jobtracker',
             action='stop'
     )
-
-    Historyserver().stop(env) #TODO HS will be separate component
-
+    
   def status(self, env):
     import status_params
     env.set_params(status_params)

ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py

@@ -106,13 +106,13 @@ def create_app_directories():
       hdfs_directory(name=params.yarn_nm_app_log_dir,
                      owner=params.yarn_user,
                      group=params.user_group,
-                     mode="1777",
+                     mode="777",
                      recursive_chmod=True
       )
     hdfs_directory(name=params.mapreduce_jobhistory_intermediate_done_dir,
                    owner=params.mapred_user,
                    group=params.user_group,
-                   mode="1777"
+                   mode="777"
     )
 
     hdfs_directory(name=params.mapreduce_jobhistory_done_dir,