AMBARI-3612. Support for client component to report status with configs (dsen)

ambari-agent/src/main/python/ambari_agent/ActionQueue.py

@@ -158,8 +158,11 @@ class ActionQueue(threading.Thread):
       if command.has_key('configurationTags'):
         configHandler.write_actual(command['configurationTags'])
         roleResult['configurationTags'] = command['configurationTags']
-
-      if command.has_key('roleCommand') and command['roleCommand'] == self.ROLE_COMMAND_START:
+      component = {'serviceName':command['serviceName'],'componentName':command['role']}
+      if command.has_key('roleCommand') and \
+        (command['roleCommand'] == self.ROLE_COMMAND_START or \
+        (command['roleCommand'] == self.ROLE_COMMAND_INSTALL \
+        and component in LiveStatus.CLIENT_COMPONENTS)):
         configHandler.copy_to_component(command['role'])
         roleResult['configurationTags'] = configHandler.read_actual_component(command['role'])
     self.commandStatuses.put_command_status(command, roleResult)

ambari-agent/src/main/python/ambari_agent/LiveStatus.py

@@ -36,6 +36,31 @@ class LiveStatus:
     "YARN", "MAPREDUCE2", "FLUME"
   ]
 
+  CLIENT_COMPONENTS = [
+    {"serviceName" : "HBASE",
+     "componentName" : "HBASE_CLIENT"},
+    {"serviceName" : "HDFS",
+     "componentName" : "HDFS_CLIENT"},
+    {"serviceName" : "MAPREDUCE",
+     "componentName" : "MAPREDUCE_CLIENT"},
+    {"serviceName" : "ZOOKEEPER",
+     "componentName" : "ZOOKEEPER_CLIENT"},
+    {"serviceName" : "OOZIE",
+     "componentName" : "OOZIE_CLIENT"},
+    {"serviceName" : "HCATALOG",
+     "componentName" : "HCAT"},
+    {"serviceName" : "HIVE",
+     "componentName" : "HIVE_CLIENT"},
+    {"serviceName" : "YARN",
+     "componentName" : "YARN_CLIENT"},
+    {"serviceName" : "MAPREDUCE2",
+     "componentName" : "MAPREDUCE2_CLIENT"},
+    {"serviceName" : "PIG",
+     "componentName" : "PIG"},
+    {"serviceName" : "SQOOP",
+     "componentName" : "SQOOP"}
+  ]
+
   COMPONENTS = [
       {"serviceName" : "HDFS",
        "componentName" : "DATANODE"},
@@ -118,29 +143,35 @@ class LiveStatus:
 
   # Live status was stripped from heartbeat after revision e1718dd
   def build(self):
-    global SERVICES, COMPONENTS, LIVE_STATUS, DEAD_STATUS
+    global SERVICES, CLIENT_COMPONENTS, COMPONENTS, LIVE_STATUS, DEAD_STATUS
     statusCheck = StatusCheck(AmbariConfig.servicesToPidNames,
       AmbariConfig.pidPathesVars, self.globalConfig,
       AmbariConfig.servicesToLinuxUser)
     livestatus = None
-    for component in self.COMPONENTS:
-      if component["serviceName"] == self.service and component["componentName"] == self.component:
-        serviceStatus = statusCheck.getStatus(component["componentName"])
+    component = {"serviceName" : self.service, "componentName" : self.component}
+    if component in self.COMPONENTS + self.CLIENT_COMPONENTS :
+      # CLIENT components can't have status STARTED
+      if component in self.CLIENT_COMPONENTS:
+        status = self.DEAD_STATUS
+      else:
+        serviceStatus = statusCheck.getStatus(self.component)
+
         if serviceStatus is None:
-          logger.warn("There is no service to pid mapping for " + component["componentName"])
+          logger.warn("There is no service to pid mapping for " + self.component)
         status = self.LIVE_STATUS if serviceStatus else self.DEAD_STATUS
-        livestatus ={"componentName" : component["componentName"],
-                       "msg" : "",
-                       "status" : status,
-                       "clusterName" : self.cluster,
-                       "serviceName" : self.service,
-                       "stackVersion": self.versionsHandler.
-                                    read_stack_version(component["componentName"])
-                    }
-        active_config = self.actualConfigHandler.read_actual_component(component['componentName'])
-        if not active_config is None:
-          livestatus['configurationTags'] = active_config
-        break
+
+      livestatus ={"componentName" : self.component,
+                   "msg" : "",
+                   "status" : status,
+                   "clusterName" : self.cluster,
+                   "serviceName" : self.service,
+                   "stackVersion": self.versionsHandler.
+                   read_stack_version(self.component)
+      }
+      active_config = self.actualConfigHandler.read_actual_component(self.component)
+      if not active_config is None:
+        livestatus['configurationTags'] = active_config
+
     logger.debug("The live status for component " + str(self.component) +\
                 " of service " + str(self.service) + " is " + str(livestatus))
     return livestatus

ambari-agent/src/test/python/TestLiveStatus.py

@@ -23,6 +23,8 @@ from ambari_agent.LiveStatus import LiveStatus
 from ambari_agent.AmbariConfig import AmbariConfig
 import socket
 import os, sys, StringIO
+from ambari_agent import ActualConfigHandler
+from mock.mock import patch, MagicMock, call
 
 class TestLiveStatus(TestCase):
 
@@ -36,7 +38,8 @@ class TestLiveStatus(TestCase):
     # enable stdout
     sys.stdout = sys.__stdout__
 
-  def test_build(self):
+  @patch.object(ActualConfigHandler.ActualConfigHandler, "read_actual_component")
+  def test_build(self, read_actual_component_mock):
     for component in LiveStatus.COMPONENTS:
       config = AmbariConfig().getConfig()
       config.set('agent', 'prefix', "dummy_files")
@@ -48,4 +51,10 @@ class TestLiveStatus(TestCase):
       if component['componentName'] == 'GANGLIA_SERVER':
         self.assertEquals(result['stackVersion'],'{"stackName":"HDP","stackVersion":"1.2.2"}',
                       'Livestatus should contain component stack version')
-  
+
+    # Test build status for CLIENT component (in LiveStatus.CLIENT_COMPONENTS)
+    read_actual_component_mock.return_value = "some tags"
+    livestatus = LiveStatus('c1', 'HDFS', 'HDFS_CLIENT', { }, config)
+    result = livestatus.build()
+    self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
+    self.assertTrue(result.has_key('configurationTags'))

ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java

@@ -187,57 +187,55 @@ public class HeartbeatMonitor implements Runnable {
         Service service = cl.getService(sch.getServiceName());
         ServiceComponent sc = service.getServiceComponent(sch
           .getServiceComponentName());
-        // Do not send status commands for client components
-        if (!sc.isClientComponent()) {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Live status will include status of service " + serviceName + " of cluster " + cl.getClusterName());
-          }
+        // Send status commands for any components
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Live status will include status of service " + serviceName + " of cluster " + cl.getClusterName());
+        }
 
-          Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
+        Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
 
-          // get the cluster config for type 'global'
-          // apply service overrides, if the tag is not the same
-          // apply host overrides, if any
+        // get the cluster config for type 'global'
+        // apply service overrides, if the tag is not the same
+        // apply host overrides, if any
 
-          Config clusterConfig = cl.getDesiredConfigByType("global");
-          if (null != clusterConfig) {
-            // cluster config for 'global'
-            Map<String, String> props = new HashMap<String, String>(clusterConfig.getProperties());
+        Config clusterConfig = cl.getDesiredConfigByType("global");
+        if (null != clusterConfig) {
+          // cluster config for 'global'
+          Map<String, String> props = new HashMap<String, String>(clusterConfig.getProperties());
 
-            // apply service overrides, only if the tag is not the same (for when service configs are overrides)
-            Config svcConfig = service.getDesiredConfigs().get("global");
-            if (null != svcConfig && !svcConfig.getVersionTag().equals(clusterConfig.getVersionTag())) {
-              props.putAll(svcConfig.getProperties());
-            }
+          // apply service overrides, only if the tag is not the same (for when service configs are overrides)
+          Config svcConfig = service.getDesiredConfigs().get("global");
+          if (null != svcConfig && !svcConfig.getVersionTag().equals(clusterConfig.getVersionTag())) {
+            props.putAll(svcConfig.getProperties());
+          }
 
-            // apply host overrides, if any
-            Host host = fsm.getHost(hostname);
-            DesiredConfig dc = host.getDesiredConfigs(cl.getClusterId()).get("global");
-            if (null != dc) {
-              Config hostConfig = cl.getConfig("global", dc.getVersion());
-              if (null != hostConfig) {
-                props.putAll(hostConfig.getProperties());
-              }
+          // apply host overrides, if any
+          Host host = fsm.getHost(hostname);
+          DesiredConfig dc = host.getDesiredConfigs(cl.getClusterId()).get("global");
+          if (null != dc) {
+            Config hostConfig = cl.getConfig("global", dc.getVersion());
+            if (null != hostConfig) {
+              props.putAll(hostConfig.getProperties());
             }
-
-            configurations.put("global", props);
           }
 
-          // HACK - if any service exists with global tag, and we have none, use
-          // that instead
-          if (configurations.isEmpty()) {
-            Config config = service.getDesiredConfigs().get("global");
-            if (null != config)
-              configurations.put("global", new HashMap<String, String>(config.getProperties()));
-          }
+          configurations.put("global", props);
+        }
 
-          StatusCommand statusCmd = new StatusCommand();
-          statusCmd.setClusterName(cl.getClusterName());
-          statusCmd.setServiceName(serviceName);
-          statusCmd.setComponentName(sch.getServiceComponentName());
-          statusCmd.setConfigurations(configurations);
-          cmds.add(statusCmd);
+        // HACK - if any service exists with global tag, and we have none, use
+        // that instead
+        if (configurations.isEmpty()) {
+          Config config = service.getDesiredConfigs().get("global");
+          if (null != config)
+            configurations.put("global", new HashMap<String, String>(config.getProperties()));
         }
+
+        StatusCommand statusCmd = new StatusCommand();
+        statusCmd.setClusterName(cl.getClusterName());
+        statusCmd.setServiceName(serviceName);
+        statusCmd.setComponentName(sch.getServiceComponentName());
+        statusCmd.setConfigurations(configurations);
+        cmds.add(statusCmd);
       }
     }
     return cmds;

ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatMonitor.java

@@ -169,7 +169,7 @@ public class TestHeartbeatMonitor {
   }
 
   @Test
-  public void testNoStatusCommandForClientComponents() throws Exception {
+  public void testStatusCommandForAnyComponents() throws Exception {
     Clusters clusters = injector.getInstance(Clusters.class);
     clusters.addHost(hostname1);
     clusters.getHost(hostname1).setOsType("centos6");
@@ -235,9 +235,11 @@ public class TestHeartbeatMonitor {
     hb.setResponseId(12);
     handler.handleHeartBeat(hb);
 
+    // HeartbeatMonitor should generate StatusCommands for
+    // MASTER, SLAVE or CLIENT components
     List<StatusCommand> cmds = hm.generateStatusCommands(hostname1);
     assertTrue("HeartbeatMonitor should generate StatusCommands for host1",
-      cmds.size() == 3);
+      cmds.size() == 4);
     assertEquals("HDFS", cmds.get(0).getServiceName());
     boolean containsDATANODEStatus = false;
     boolean containsNAMENODEStatus = false;
@@ -255,11 +257,12 @@ public class TestHeartbeatMonitor {
     assertTrue(containsDATANODEStatus);
     assertTrue(containsNAMENODEStatus);
     assertTrue(containsSECONDARY_NAMENODEStatus);
-    assertFalse(containsHDFS_CLIENTStatus);
+    assertTrue(containsHDFS_CLIENTStatus);
 
     cmds = hm.generateStatusCommands(hostname2);
-    assertTrue("HeartbeatMonitor should not generate StatusCommands for host2" +
-      " because it has only client components", cmds.isEmpty());
+    assertTrue("HeartbeatMonitor should generate StatusCommands for host2, " +
+      "even if it has only client components", cmds.size() == 1);
+    assertTrue(cmds.get(0).getComponentName().equals(Role.HDFS_CLIENT.name()));
   }
 
   @Test