AMBARI-2809. Add support for install, start and smoke test JournalNode slave Component in HDFS.

ambari-agent/src/main/puppet/modules/hdp-hadoop/files/checkWebUI.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import optparse
+import urllib
+
+#
+# Main.
+#
+def main():
+  parser = optparse.OptionParser(usage="usage: %prog [options] component ")
+  parser.add_option("-u", "--url", dest="url", help="Host:Port for WEB UI to check it availability")
+
+  (options, args) = parser.parse_args()
+  
+  url = options.url
+
+  try:
+    httpCode = urllib.urlopen('http://' + url).getcode()
+  except Exception:
+    httpCode = 404
+
+  if httpCode != 200:
+    exit(1)
+  else:
+    exit(0)
+
+if __name__ == "__main__":
+  main()

ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/hdfs/service_check.pp

@@ -62,14 +62,7 @@ class hdp-hadoop::hdfs::service_check()
     notify    => Hdp-hadoop::Exec-hadoop['hdfs::service_check::test']
   }
 
-  hdp-hadoop::exec-hadoop { 'hdfs::service_check::test':
-    command     => $test_cmd,
-    refreshonly => true,
-    user      => $hdp::params::smokeuser,
-    require     => Hdp-hadoop::Exec-hadoop['hdfs::service_check::create_file'],
-    #notify      => Hdp-hadoop::Exec-hadoop['hdfs::service_check::cleanup']  #TODO: put in after testing
-    before      => Anchor['hdp-hadoop::hdfs::service_check::end'] #TODO: remove after testing
-  }
+
 
    #TODO: put in after testing
  #  hdp-hadoop::exec-hadoop { 'hdfs::service_check::cleanup':
@@ -78,6 +71,61 @@ class hdp-hadoop::hdfs::service_check()
  #   require     => Hdp-hadoop::Exec-hadoop['hdfs::service_check::test'],
  #   before      => Anchor['hdp-hadoop::hdfs::service_check::end']
   #}
+  
+  if hdp_is_empty($hdp::params::journalnode_hosts) {
+    ##No journalnode hosts, just run hdfs test
+    hdp-hadoop::exec-hadoop { 'hdfs::service_check::test':
+      command     => $test_cmd,
+      refreshonly => true,
+      user      => $hdp::params::smokeuser,
+      require     => Hdp-hadoop::Exec-hadoop['hdfs::service_check::create_file'],
+      before      => Anchor['hdp-hadoop::hdfs::service_check::end']
+    } 
+  }
+  else {
+  ## Cluster has journalnode hosts, run hdfs test and test of journalnodes
+    hdp-hadoop::exec-hadoop { 'hdfs::service_check::test':
+      command     => $test_cmd,
+      refreshonly => true,
+      user      => $hdp::params::smokeuser,
+      require     => Hdp-hadoop::Exec-hadoop['hdfs::service_check::create_file'],
+      before      => Class['hdp-hadoop::journalnode::service_check']
+    } 
+    class { 'hdp-hadoop::journalnode::service_check':
+      journalnode_host => $hdp::params::journalnode_hosts,
+      require          => Hdp-hadoop::Exec-hadoop['hdfs::service_check::test'],
+      before           => Anchor['hdp-hadoop::hdfs::service_check::end']
+    }
+  }
+  
+
   anchor{ 'hdp-hadoop::hdfs::service_check::end':}
 
 }
+
+class hdp-hadoop::journalnode::service_check($journalnode_host)
+{
+  
+  $journalnode_port = $hdp::params::journalnode_port
+  $smoke_test_user = $hdp::params::smokeuser
+  
+  $checkWebUIFileName = "checkWebUI.py"
+  $checkWebUIFilePath = "/tmp/$checkWebUIFileName"
+
+  $checkWebUICmd = "su - ${smoke_test_user} -c 'python $checkWebUIFilePath -u $journalnode_host:$journalnode_port'"
+
+  file { $checkWebUIFilePath:
+    ensure => present,
+    source => "puppet:///modules/hdp-hadoop/$checkWebUIFileName",
+    mode => '0755'
+  }
+
+  exec { $checkWebUIFilePath:
+    command   => $checkWebUICmd,
+    tries     => 3,
+    try_sleep => 5,
+    path      => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+    logoutput => "true"
+}
+  anchor{"hdp-hadoop::smoketest::begin":} -> File[$checkWebUIFilePath] -> Exec[$checkWebUIFilePath] -> anchor{"hdp-hadoop::smoketest::end":}
+}

ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/journalnode.pp

@@ -0,0 +1,60 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp-hadoop::journalnode(
+  $service_state = $hdp::params::cluster_service_state,
+  $opts = {}
+) inherits hdp-hadoop::params 
+{
+
+  $hdp::params::service_exists['hdp-hadoop::journalnode'] = true
+  
+  Hdp-hadoop::Common<||>{service_states +> $service_state}
+  Hdp-hadoop::Package<||>{include_64_bit => true}
+  Hdp-hadoop::Configfile<||>{sizes +> 64}
+  
+  $jn_edits_dir = $hdp-hadoop::params::jn_edits_dir
+  $hdfs_user = $hdp-hadoop::params::hdfs_user
+  
+  if ($service_state == 'no_op') {
+  } elsif ($service_state in ['running','stopped','installed_and_configured','uninstalled']) { 
+  
+    #adds package, users and directories, and common hadoop configs
+    include hdp-hadoop::initialize
+    
+    hdp::directory_recursive_create{ $jn_edits_dir:
+        service_state => $service_state,
+        force => true,
+        owner => $hdfs_user
+      }
+      
+    hdp-hadoop::service{ 'journalnode':
+      ensure         => $service_state,
+      user           => $hdp-hadoop::params::hdfs_user,
+      create_pid_dir => true,
+      create_log_dir => true
+    }
+    
+    #top level does not need anchors
+    Anchor['hdp-hadoop::begin'] -> Hdp::Directory_recursive_create[$jn_edits_dir] -> Hdp-hadoop::Service['journalnode'] -> Anchor['hdp-hadoop::end'] 
+  } else {
+    hdp_fail("TODO not implemented yet: service_state = ${service_state}")
+  }
+}

ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp

@@ -131,6 +131,9 @@ class hdp-hadoop::params(
   $dfs_support_append = hdp_default("hdfs-site/dfs.support.append",true)
 
   $dfs_webhdfs_enabled = hdp_default("hdfs-site/dfs.webhdfs.enabled",false)
+  
+  $jn_edits_dir = hdp_default("hdfs-site/dfs.journalnode.edits.dir", "/grid/0/hdfs/journal")
+  
 
 
  ######### mapred #######

ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp

@@ -33,6 +33,7 @@ define hdp::configfile(
   $nm_hosts = $hdp::params::nm_hosts,
   $hs_host = $hdp::params::hs_host,
   $slave_hosts = $hdp::params::slave_hosts,
+  $journalnode_hosts = $hdp::params::journalnode_hosts,
   $mapred_tt_hosts = $hdp::params::mapred_tt_hosts,
   $all_hosts = $hdp::params::all_hosts,
   $hbase_rs_hosts = $hdp::params::hbase_rs_hosts,
@@ -53,6 +54,7 @@ define hdp::configfile(
   $public_rm_host = $hdp::params::public_rm_host,
   $public_nm_hosts = $hdp::params::public_nm_hosts,
   $public_hs_host = $hdp::params::public_hs_host,
+  $public_journalnode_hosts = $hdp::params::public_journalnode_hosts,
   $public_jtnode_host = $hdp::params::public_jtnode_host,
   $public_hbase_master_hosts = $hdp::params::public_hbase_master_hosts,
   $public_zookeeper_hosts = $hdp::params::public_zookeeper_hosts,

ambari-agent/src/main/puppet/modules/hdp/manifests/init.pp

@@ -39,10 +39,12 @@ class hdp(
     $namenode_port = hdp_get_port_from_url($hdfs-site["dfs.http.address"])
     $snamenode_port = hdp_get_port_from_url($hdfs-site["dfs.secondary.http.address"])
     $datanode_port = hdp_get_port_from_url($hdfs-site["dfs.datanode.http.address"])
+    $journalnode_port = hdp_get_port_from_url($hdfs-site["dfs.journalnode.http-address"])
   } else {
     $namenode_port = "50070"
     $snamenode_port = "50090"
     $datanode_port = "50075"
+    $journalnode_port = "8480"
   }
 
   if has_key($configuration, 'mapred-site') {

ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp

@@ -64,6 +64,7 @@ class hdp::params()
   $snamenode_host = hdp_default("snamenode_host")
   $jtnode_host = hdp_default("jtnode_host")
   $slave_hosts = hdp_default("slave_hosts")
+  $journalnode_hosts = hdp_default("journalnode_hosts")
 
   $nn_principal_str = hdp_default("hdfs-site/dfs.namenode.kerberos.principal", "nn/_HOST@EXAMPLE.COM")
   if ("_HOST" in $nn_principal_str and hdp_is_empty($namenode_host) == false) {
@@ -154,6 +155,7 @@ class hdp::params()
     $public_rm_host = hdp_host_attribute($hostAttributes,"publicfqdn",$rm_host)
     $public_nm_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$nm_hosts)
     $public_hs_host = hdp_host_attribute($hostAttributes,"publicfqdn",$hs_host)
+    $public_journalnode_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$journalnode_hosts)
     $public_jtnode_host = hdp_host_attribute($hostAttributes,"publicfqdn",$jtnode_host)
     $public_hbase_master_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$hbase_master_hosts)
     $public_zookeeper_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$zookeeper_hosts)
@@ -169,6 +171,7 @@ class hdp::params()
     $public_rm_host = hdp_default("rm_host")
     $public_nm_hosts = hdp_default("nm_hosts")
     $public_hs_host = hdp_default("hs_host")
+    $public_journalnode_hosts = hdp_default("journalnode_hosts")
     $public_jtnode_host = hdp_default("jtnode_host")
     $public_hbase_master_hosts = hdp_default("hbase_master_hosts")
     $public_zookeeper_hosts = hdp_default("zookeeper_hosts")

ambari-agent/src/main/python/ambari_agent/AmbariConfig.py

@@ -134,7 +134,9 @@ rolesToClass = {
   'HISTORYSERVER_SERVICE_CHECK': 'hdp-yarn::historyserver::service_check',
   'TEZ_CLIENT': 'hdp-tez::tez_client',
   'YARN_SERVICE_CHECK': 'hdp-yarn::yarn::service_check',
-  'FLUME_SERVER': 'hdp-flume'
+  'FLUME_SERVER': 'hdp-flume',
+  'JOURNALNODE': 'hdp-hadoop::journalnode',
+  'JOURNALNODE_SERVICE_CHECK': 'hdp-hadoop::journalnode::service_check'
 }
 
 serviceStates = {
@@ -153,6 +155,7 @@ servicesToPidNames = {
   'RESOURCEMANAGER': 'yarn-{USER}-resourcemanager.pid$',
   'NODEMANAGER': 'yarn-{USER}-nodemanager.pid$',
   'HISTORYSERVER': 'mapred-{USER}-historyserver.pid$',
+  'JOURNALNODE': 'hadoop-{USER}-journalnode.pid$',
   'OOZIE_SERVER': 'oozie.pid',
   'ZOOKEEPER_SERVER': 'zookeeper_server.pid',
   'FLUME_SERVER': 'flume-node.pid',
@@ -176,6 +179,7 @@ servicesToLinuxUser = {
   'NAMENODE': 'hdfs_user',
   'SECONDARY_NAMENODE': 'hdfs_user',
   'DATANODE': 'hdfs_user',
+  'JOURNALNODE': 'hdfs_user',
   'JOBTRACKER': 'mapred_user',
   'TASKTRACKER': 'mapred_user',
   'RESOURCEMANAGER': 'yarn_user',

ambari-agent/src/main/python/ambari_agent/LiveStatus.py

@@ -43,6 +43,8 @@ class LiveStatus:
        "componentName" : "NAMENODE"},
       {"serviceName" : "HDFS",
        "componentName" : "SECONDARY_NAMENODE"},
+      {"serviceName" : "HDFS",
+       "componentName" : "JOURNALNODE"},
 
       {"serviceName" : "MAPREDUCE",
        "componentName" : "JOBTRACKER"},

ambari-server/src/main/java/org/apache/ambari/server/Role.java

@@ -82,5 +82,6 @@ public enum Role {
   YARN_CLIENT,
   HISTORYSERVER,
   TEZ_CLIENT,
-  FLUME_SERVER
+  FLUME_SERVER,
+  JOURNALNODE
 }

ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java

@@ -89,6 +89,7 @@ public class StageUtils {
     componentToClusterInfoKeyMap.put("RESOURCEMANAGER", "rm_host");
     componentToClusterInfoKeyMap.put("NODEMANAGER", "nm_hosts");
     componentToClusterInfoKeyMap.put("HISTORYSERVER", "hs_host");
+    componentToClusterInfoKeyMap.put("JOURNALNODE", "journalnode_hosts");
     componentToClusterInfoKeyMap.put("ZOOKEEPER_SERVER", "zookeeper_hosts");
     componentToClusterInfoKeyMap.put("FLUME_SERVER", "flume_hosts");
     componentToClusterInfoKeyMap.put("HBASE_MASTER", "hbase_master_hosts");

ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/configuration/hdfs-site.xml

@@ -455,5 +455,20 @@ don't exist, they will be created with this permission.</description>
     <value>30000</value>
     <description>Datanode is stale after not getting a heartbeat in this interval in ms</description>
   </property>
+  
+  <property>
+    <name>dfs.journalnode.http-address</name>
+    <value>0.0.0.0:8480</value>
+    <description>The address and port the JournalNode web UI listens on.
+     If the port is 0 then the server will start on a free port. </description>
+  </property>
+  
+  <property>
+    <name>dfs.journalnode.edits.dir</name>
+    <value>/grid/0/hdfs/journal</value>
+    <description>The path where the JournalNode daemon will store its local state. </description>
+  </property>
+  
+  
 
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.0.3/services/HDFS/metainfo.xml

@@ -40,6 +40,12 @@
             <name>HDFS_CLIENT</name>
             <category>CLIENT</category>
         </component>
+        
+        <component>
+            <name>JOURNALNODE</name>
+            <category>SLAVE</category>
+        </component>
+        
     </components>