AMBARI-5148. LogViewer service needs to run on all supervisor nodes (aonishuk)

ambari-agent/src/main/python/ambari_agent/LiveStatus.py

@@ -143,9 +143,7 @@ class LiveStatus:
       {"serviceName" : "STORM",
        "componentName" : "STORM_UI_SERVER"},
       {"serviceName" : "STORM",
-       "componentName" : "DRPC_SERVER"},
-      {"serviceName" : "STORM",
-       "componentName" : "LOGVIEWER_SERVER"}
+       "componentName" : "DRPC_SERVER"}
   ]
 
   LIVE_STATUS = "STARTED"

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/scripts/params.py

@@ -69,7 +69,6 @@ hbase_rs_port = "60030"
 storm_ui_port = config['configurations']['storm-site']['ui.port']
 drpc_port = config['configurations']['storm-site']['drpc.port']
 nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
-logviewer_port = config['configurations']['storm-site']['logviewer.port']
 supervisor_port = "56431"
 storm_rest_api_port = "8745"
 falcon_port = config['configurations']['global']['falcon_port']
@@ -147,7 +146,6 @@ _ganglia_server_host = default("/clusterHostInfo/ganglia_server_host",None)
 _app_timeline_server_hosts = default("/clusterHostInfo/app_timeline_server_hosts",None)
 _nimbus_host = default("/clusterHostInfo/nimbus_hosts",None)
 _drpc_host = default("/clusterHostInfo/drpc_server_hosts",None)
-_logwier_host = default("/clusterHostInfo/logviewer_server_hosts",None)
 _supervisor_hosts = default("/clusterHostInfo/supervisor_hosts",None)
 _storm_ui_host = default("/clusterHostInfo/storm_ui_server_hosts",None)
 _storm_rest_api_hosts = default("/clusterHostInfo/storm_rest_api_hosts",None)
@@ -188,7 +186,6 @@ hostgroup_defs = {
     'journalnodes' : _journalnode_hosts,
     'nimbus' : _nimbus_host,
     'drpc-server' : _drpc_host,
-    'logviewer-server' : _logwier_host,
     'storm_ui' : _storm_ui_host,
     'supervisors' : _supervisor_hosts,
     'storm_rest_api' : _storm_rest_api_hosts,

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-servicegroups.cfg.j2

@@ -80,7 +80,6 @@ define servicegroup {
 {% endif %}
 {% if hostgroup_defs['nimbus'] or
   hostgroup_defs['drpc-server'] or
-  hostgroup_defs['logviewer-server'] or
   hostgroup_defs['storm_ui'] or
   hostgroup_defs['supervisors'] or
   hostgroup_defs['storm_rest_api']%}

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2

@@ -288,22 +288,8 @@ define service {
 }
 {% endif %}
 
-{% if hostgroup_defs['logviewer-server'] %}
-# logviewer Checks
-define service {
-        hostgroup_name          logviewer-server
-        use                     hadoop-service
-        service_description     LOGVIEWER_SERVER::Logviewer Server process
-        servicegroups           STORM
-        check_command           check_tcp_wrapper!{{ logviewer_port }}!-w 1 -c 1
-        normal_check_interval   0.5
-        retry_check_interval    0.25
-        max_check_attempts      3
-}
-{% endif %}
-
 {% if hostgroup_defs['storm_rest_api'] %}
-# logviewer Checks
+# Storm REST API Checks
 define service {
         hostgroup_name          storm_rest_api
         use                     hadoop-service

ambari-server/src/main/resources/stacks/HDP/2.1/role_command_order.json

@@ -10,7 +10,6 @@
     "STORM_UI_SERVER-START" : ["NIMBUS-START"],
     "DRPC_SERVER-START" : ["NIMBUS-START"],
     "STORM_REST_API-START" : ["NIMBUS-START", "STORM_UI_SERVER-START", "SUPERVISOR-START", "DRPC_SERVER-START"],
-    "LOGVIEWER_SERVER-START" : ["NIMBUS-START"],
     "HBASE_MASTER-START": ["ZOOKEEPER_SERVER-START"],
     "HBASE_REGIONSERVER-START": ["HBASE_MASTER-START"],
     "OOZIE_SERVER-START": ["JOBTRACKER-START", "TASKTRACKER-START"],
@@ -36,10 +35,10 @@
     "ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
     "ZOOKEEPER_QUORUM_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
     "STORM_SERVICE_CHECK-SERVICE_CHECK": ["NIMBUS-START", "SUPERVISOR-START", "STORM_UI_SERVER-START",
-        "DRPC_SERVER-START", "LOGVIEWER_SERVER-START"],
+        "DRPC_SERVER-START"],
     "ZOOKEEPER_SERVER-STOP" : ["HBASE_MASTER-STOP", "HBASE_REGIONSERVER-STOP"],
     "HBASE_MASTER-STOP": ["HBASE_REGIONSERVER-STOP"],
-    "NIMBUS-STOP" : ["SUPERVISOR-STOP", "STORM_UI_SERVER-STOP", "DRPC_SERVER-STOP", "LOGVIEWER_SERVER-STOP"],
+    "NIMBUS-STOP" : ["SUPERVISOR-STOP", "STORM_UI_SERVER-STOP", "DRPC_SERVER-STOP"],
     "TASKTRACKER-UPGRADE": ["JOBTRACKER-UPGRADE"],
     "MAPREDUCE_CLIENT-UPGRADE": ["TASKTRACKER-UPGRADE", "JOBTRACKER-UPGRADE"],
     "ZOOKEEPER_SERVER-UPGRADE": ["MAPREDUCE_CLIENT-UPGRADE"],

ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/metainfo.xml

@@ -74,19 +74,8 @@
             <timeout>600</timeout>
           </commandScript>
         </component>
-
-        <component>
-          <name>LOGVIEWER_SERVER</name>
-          <category>MASTER</category>
-          <commandScript>
-            <script>scripts/logviewer_server.py</script>
-            <scriptType>PYTHON</scriptType>
-            <timeout>600</timeout>
-          </commandScript>
-        </component>
       </components>
 
-
       <osSpecifics>
         <osSpecific>
           <osType>any</osType>

ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/logviewer_server.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import sys
-from resource_management import *
-from storm import storm
-from service import service
-from service_check import ServiceCheck
-
-
-class LogviewerServer(Script):
-  def install(self, env):
-    self.install_packages(env)
-    self.configure(env)
-
-  def configure(self, env):
-    import params
-    env.set_params(params)
-
-    storm()
-
-  def start(self, env):
-    import params
-    env.set_params(params)
-    self.configure(env)
-
-    service("logviewer", action="start")
-
-  def stop(self, env):
-    import params
-    env.set_params(params)
-
-    service("logviewer", action="stop")
-
-  def status(self, env):
-    import status_params
-    env.set_params(status_params)
-    check_process_status(status_params.pid_logviewer)
-
-if __name__ == "__main__":
-  LogviewerServer().execute()

ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/supervisor.py

@@ -41,12 +41,14 @@ class Supervisor(Script):
     self.configure(env)
 
     service("supervisor", action="start")
+    service("logviewer", action="start")
 
   def stop(self, env):
     import params
     env.set_params(params)
 
     service("supervisor", action="stop")
+    service("logviewer", action="stop")
 
   def status(self, env):
     import status_params

ambari-server/src/test/python/stacks/2.1/STORM/test_storm_logviewer_server.py

@@ -1,194 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-from mock.mock import MagicMock, call, patch
-from stacks.utils.RMFTestCase import *
-import  resource_management.core.source
-
-@patch.object(resource_management.core.source, "InlineTemplate", new = MagicMock(return_value='InlineTemplateMock'))
-class TestStormLogviewerServer(RMFTestCase):
-
-  def test_configure_default(self):
-    self.executeScript("2.1/services/STORM/package/scripts/logviewer_server.py",
-                       classname = "LogviewerServer",
-                       command = "configure",
-                       config_file="default.json"
-    )
-    self.assert_configure_default()
-    self.assertNoMoreResources()
-
-  def test_start_default(self):
-    self.executeScript("2.1/services/STORM/package/scripts/logviewer_server.py",
-                       classname = "LogviewerServer",
-                       command = "start",
-                       config_file="default.json"
-    )
-
-    self.assert_configure_default()
-
-    self.assertResourceCalled('Execute', 'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 PATH=$PATH:/usr/jdk64/jdk1.7.0_45/bin /usr/bin/storm logviewer',
-      wait_for_finish = False,
-      not_if = 'ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1',
-      user = 'storm',
-    )
-
-    self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.logviewer$" && pgrep -f "^java.+backtype.storm.daemon.logviewer$" > /var/run/storm/logviewer.pid',
-      logoutput = True,
-      tries = 6,
-      user = 'storm',
-      try_sleep = 10,
-    )
-
-    self.assertNoMoreResources()
-
-  def test_stop_default(self):
-    self.executeScript("2.1/services/STORM/package/scripts/logviewer_server.py",
-                       classname = "LogviewerServer",
-                       command = "stop",
-                       config_file="default.json"
-    )
-    self.assertResourceCalled('Execute', 'kill `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1',
-                              not_if = '! (ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1)',
-                              )
-    self.assertResourceCalled('Execute', 'kill -9 `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1',
-                              not_if = 'sleep 2; ! (ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1) || sleep 20; ! (ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1)',
-                              ignore_failures=True
-                              )
-    self.assertResourceCalled('Execute', 'rm -f /var/run/storm/logviewer.pid')
-    self.assertNoMoreResources()
-
-  def test_configure_default(self):
-    self.executeScript("2.1/services/STORM/package/scripts/logviewer_server.py",
-                       classname = "LogviewerServer",
-                       command = "configure",
-                       config_file="secured.json"
-    )
-    self.assert_configure_secured()
-    self.assertNoMoreResources()
-
-  def test_start_secured(self):
-    self.executeScript("2.1/services/STORM/package/scripts/logviewer_server.py",
-                       classname = "LogviewerServer",
-                       command = "start",
-                       config_file="secured.json"
-    )
-
-    self.assert_configure_secured()
-
-    self.assertResourceCalled('Execute', 'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 PATH=$PATH:/usr/jdk64/jdk1.7.0_45/bin /usr/bin/storm logviewer',
-      wait_for_finish = False,
-      not_if = 'ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1',
-      user = 'storm',
-    )
-
-    self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.logviewer$" && pgrep -f "^java.+backtype.storm.daemon.logviewer$" > /var/run/storm/logviewer.pid',
-      logoutput = True,
-      tries = 6,
-      user = 'storm',
-      try_sleep = 10,
-    )
-
-    self.assertNoMoreResources()
-
-  def test_stop_secured(self):
-    self.executeScript("2.1/services/STORM/package/scripts/logviewer_server.py",
-                       classname = "LogviewerServer",
-                       command = "stop",
-                       config_file="secured.json"
-    )
-    self.assertResourceCalled('Execute', 'kill `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1',
-                              not_if = '! (ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1)'
-                              )
-    self.assertResourceCalled('Execute', 'kill -9 `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1',
-                              not_if = 'sleep 2; ! (ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1) || sleep 20; ! (ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1)',
-                              ignore_failures=True
-    )
-    self.assertResourceCalled('Execute', 'rm -f /var/run/storm/logviewer.pid')
-    self.assertNoMoreResources()
-
-  def assert_configure_default(self):
-
-    self.assertResourceCalled('Directory', '/var/log/storm',
-      owner = 'storm',
-      group = 'hadoop',
-      recursive = True,
-    )
-    self.assertResourceCalled('Directory', '/var/run/storm',
-      owner = 'storm',
-      group = 'hadoop',
-      recursive = True,
-    )
-    self.assertResourceCalled('Directory', '/hadoop/storm',
-      owner = 'storm',
-      group = 'hadoop',
-      recursive = True,
-    )
-    self.assertResourceCalled('Directory', '/etc/storm/conf',
-      owner = 'storm',
-      group = 'hadoop',
-      recursive = True,
-    )
-    self.assertResourceCalled('File', '/etc/storm/conf/config.yaml',
-      owner = 'storm',
-      content = Template('config.yaml.j2'),
-      group = 'hadoop',
-    )
-    self.assertResourceCalled('File', '/etc/storm/conf/storm.yaml',
-      owner = 'storm',
-      content = 'InlineTemplateMock',
-      group = 'hadoop',
-      mode = None,
-    )
-
-  def assert_configure_secured(self):
-    self.assertResourceCalled('Directory', '/var/log/storm',
-      owner = 'storm',
-      group = 'hadoop',
-      recursive = True,
-    )
-    self.assertResourceCalled('Directory', '/var/run/storm',
-      owner = 'storm',
-      group = 'hadoop',
-      recursive = True,
-    )
-    self.assertResourceCalled('Directory', '/hadoop/storm',
-      owner = 'storm',
-      group = 'hadoop',
-      recursive = True,
-    )
-    self.assertResourceCalled('Directory', '/etc/storm/conf',
-      owner = 'storm',
-      group = 'hadoop',
-      recursive = True,
-    )
-    self.assertResourceCalled('File', '/etc/storm/conf/config.yaml',
-      owner = 'storm',
-      content = Template('config.yaml.j2'),
-      group = 'hadoop',
-    )
-    self.assertResourceCalled('File', '/etc/storm/conf/storm.yaml',
-      owner = 'storm',
-      content = 'InlineTemplateMock',
-      group = 'hadoop',
-      mode = None,
-    )
-    self.assertResourceCalled('TemplateConfig', '/etc/storm/conf/storm_jaas.conf',
-      owner = 'storm',
-    )

ambari-server/src/test/python/stacks/2.1/STORM/test_storm_supervisor.py

@@ -55,6 +55,18 @@ class TestStormSupervisor(RMFTestCase):
       user = 'storm',
       try_sleep = 10,
     )
+    self.assertResourceCalled('Execute', 'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 PATH=$PATH:/usr/jdk64/jdk1.7.0_45/bin /usr/bin/storm logviewer',
+                               wait_for_finish = False,
+                               not_if = 'ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1',
+                               user = 'storm'
+    )
+
+    self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.logviewer$" && pgrep -f "^java.+backtype.storm.daemon.logviewer$" > /var/run/storm/logviewer.pid',
+                              logoutput = True,
+                              tries = 6,
+                              user = 'storm',
+                              try_sleep = 10
+    )
 
     self.assertNoMoreResources()
 
@@ -72,6 +84,14 @@ class TestStormSupervisor(RMFTestCase):
                               ignore_failures=True
     )
     self.assertResourceCalled('Execute', 'rm -f /var/run/storm/supervisor.pid')
+    self.assertResourceCalled('Execute', 'kill `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1',
+                              not_if = '! (ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1)')
+
+    self.assertResourceCalled('Execute', 'kill -9 `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1',
+                              not_if = 'sleep 2; ! (ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1) || sleep 20; ! (ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1)',
+                              ignore_failures = True)
+
+    self.assertResourceCalled('Execute', 'rm -f /var/run/storm/logviewer.pid')
     self.assertNoMoreResources()
 
   def test_configure_default(self):
@@ -104,7 +124,18 @@ class TestStormSupervisor(RMFTestCase):
       user = 'storm',
       try_sleep = 10,
     )
+    self.assertResourceCalled('Execute', 'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 PATH=$PATH:/usr/jdk64/jdk1.7.0_45/bin /usr/bin/storm logviewer',
+                        wait_for_finish = False,
+                        not_if = 'ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1',
+                        user = 'storm'
+    )
 
+    self.assertResourceCalled('Execute', 'pgrep -f "^java.+backtype.storm.daemon.logviewer$" && pgrep -f "^java.+backtype.storm.daemon.logviewer$" > /var/run/storm/logviewer.pid',
+                        logoutput = True,
+                        tries = 6,
+                        user = 'storm',
+                        try_sleep = 10
+    )
     self.assertNoMoreResources()
 
   def test_stop_secured(self):
@@ -121,6 +152,14 @@ class TestStormSupervisor(RMFTestCase):
                               ignore_failures=True
     )
     self.assertResourceCalled('Execute', 'rm -f /var/run/storm/supervisor.pid')
+    self.assertResourceCalled('Execute', 'kill `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1',
+                              not_if = '! (ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1)')
+
+    self.assertResourceCalled('Execute', 'kill -9 `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1',
+                              not_if = 'sleep 2; ! (ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1) || sleep 20; ! (ls /var/run/storm/logviewer.pid >/dev/null 2>&1 && ps `cat /var/run/storm/logviewer.pid` >/dev/null 2>&1)',
+                              ignore_failures = True)
+
+    self.assertResourceCalled('Execute', 'rm -f /var/run/storm/logviewer.pid')
     self.assertNoMoreResources()
 
   def assert_configure_default(self):

ambari-server/src/test/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml

@@ -75,15 +75,6 @@
           </commandScript>
         </component>
 
-        <component>
-          <name>LOGVIEWER_SERVER</name>
-          <category>MASTER</category>
-          <commandScript>
-            <script>scripts/logviewer_server.py</script>
-            <scriptType>PYTHON</scriptType>
-            <timeout>600</timeout>
-          </commandScript>
-        </component>
       </components>
 
 

contrib/addons/src/addOns/nagios/scripts/nagios_alerts.php

@@ -403,7 +403,6 @@ function hdp_mon_generate_response( $response_data )
       case "STORM_UI_SERVER":
       case "NIMBUS":
       case "DRPC_SERVER":
-      case "LOGVIEWER_SERVER":
       case "SUPERVISOR":
       case "STORM_REST_API":
         $pieces[0] = "STORM";