AMBARI-4579. Add Storm REST API service to stack definition for

Managing/Configuration. (Arsen Babych via aonishuk)

ambari-server/src/main/resources/stacks/HDP/2.1.1/role_command_order.json

@@ -9,6 +9,7 @@
     "SUPERVISOR-START" : ["NIMBUS-START"],
     "STORM_UI_SERVER-START" : ["NIMBUS-START"],
     "DRPC_SERVER-START" : ["NIMBUS-START"],
+    "STORM_REST_API-START" : ["NIMBUS-START", "STORM_UI_SERVER-START", "SUPERVISOR-START", "DRPC_SERVER-START"],
     "LOGVIEWER_SERVER-START" : ["NIMBUS-START"],
     "HBASE_MASTER-START": ["ZOOKEEPER_SERVER-START"],
     "HBASE_REGIONSERVER-START": ["HBASE_MASTER-START"],
@@ -34,7 +35,8 @@
     "SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"],
     "ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
     "ZOOKEEPER_QUORUM_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
-    "STORM_SERVICE_CHECK-SERVICE_CHECK": ["NIMBUS-START", "SUPERVISOR-START"],
+    "STORM_SERVICE_CHECK-SERVICE_CHECK": ["NIMBUS-START", "SUPERVISOR-START", "STORM_UI_SERVER-START",
+        "DRPC_SERVER-START", "LOGVIEWER_SERVER-START"],
     "ZOOKEEPER_SERVER-STOP" : ["HBASE_MASTER-STOP", "HBASE_REGIONSERVER-STOP"],
     "HBASE_MASTER-STOP": ["HBASE_REGIONSERVER-STOP"],
     "NIMBUS-STOP" : ["SUPERVISOR-STOP", "STORM_UI_SERVER-STOP", "DRPC_SERVER-STOP", "LOGVIEWER_SERVER-STOP"],

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/metainfo.xml

@@ -35,6 +35,16 @@
           </commandScript>
         </component>
 
+        <component>
+          <name>STORM_REST_API</name>
+          <category>MASTER</category>
+          <commandScript>
+            <script>scripts/rest_api.py</script>
+            <scriptType>PYTHON</scriptType>
+            <timeout>600</timeout>
+          </commandScript>
+        </component>
+
         <component>
           <name>SUPERVISOR</name>
           <category>SLAVE</category>

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/params.py

@@ -32,3 +32,9 @@ local_dir = config['configurations']['storm-site']['storm.local.dir']
 user_group = config['configurations']['global']['user_group']
 java64_home = config['hostLevelParams']['java_home']
 nimbus_host = config['configurations']['storm-site']['nimbus.host']
+nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
+nimbus_host = config['configurations']['storm-site']['nimbus.host']
+rest_api_port = "8745"
+rest_api_admin_port = "8746"
+rest_api_conf_file = format("{conf_dir}/config.yaml")
+rest_lib_dir = "/usr/lib/storm/contrib/storm-rest"

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/rest_api.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from storm import storm
+from service import service
+from service_check import ServiceCheck
+
+
+class RestApi(Script):
+  def install(self, env):
+    self.install_packages(env)
+    self.configure(env)
+
+  def configure(self, env):
+    import params
+    env.set_params(params)
+
+    storm()
+
+  def start(self, env):
+    import params
+    env.set_params(params)
+    self.configure(env)
+
+    service("rest_api", action="start")
+
+  def stop(self, env):
+    import params
+    env.set_params(params)
+
+    service("rest_api", action="stop")
+
+  def status(self, env):
+    import status_params
+    env.set_params(status_params)
+    check_process_status(status_params.pid_rest_api)
+
+if __name__ == "__main__":
+  RestApi().execute()

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/service.py

@@ -33,24 +33,29 @@ def service(
   pid_file = status_params.pid_files[name]
   no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
 
-  if action == "start":
-    cmd = ["/usr/bin/storm", name]
-    if name == "ui":
-      crt_pid_cmd = format("pgrep -f \"^java.+backtype.storm.ui.core$\" > {pid_file}")
-    else :
-      crt_pid_cmd = format("pgrep -f \"^java.+backtype.storm.daemon.{name}$\" > {pid_file}")
+  if name == 'ui':
+    process_cmd = "^java.+backtype.storm.ui.core$"
+  elif name == "rest_api":
+    process_cmd = format("java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server")
+  else:
+    process_cmd = format("^java.+backtype.storm.daemon.{name}$")
 
-    #Execute(cmd,
-    #        not_if=no_op_test,
-    #        user=params.storm_user
-    #)
+  crt_pid_cmd = format("pgrep -f \"{process_cmd}\" > {pid_file}")
 
-    #TODO run from storm user
+  if action == "start":
+    if name == "rest_api":
+      cmd = format("env PATH=$PATH:{java64_home}/bin {process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
+    else:
+      cmd = format("env PATH=$PATH:{java64_home}/bin /usr/bin/storm {name}")
 
-    if call(no_op_test)[0]:
-      subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env={"PATH":format("{java64_home}/bin:/bin")})
+    Execute(cmd,
+           not_if=no_op_test,
+           user=params.storm_user,
+           wait_for_finish=False
+    )
 
     Execute(crt_pid_cmd,
+            user=params.storm_user,
             logoutput=True,
             tries=6,
             try_sleep=10

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/status_params.py

@@ -27,9 +27,10 @@ pid_supervisor = format("{pid_dir}/supervisor.pid")
 pid_drpc = format("{pid_dir}/drpc.pid")
 pid_ui = format("{pid_dir}/ui.pid")
 pid_logviewer = format("{pid_dir}/logviewer.pid")
-
+pid_rest_api = format("{pid_dir}/restapi.pid")
 pid_files = {"logviewer":pid_logviewer,
              "ui": pid_ui,
              "nimbus": pid_nimbus,
              "supervisor": pid_supervisor,
-             "drpc": pid_drpc}
+             "drpc": pid_drpc,
+             "rest_api": pid_rest_api}

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/scripts/storm.py

@@ -31,6 +31,12 @@ def storm():
             recursive=True
   )
 
+  File(format("{conf_dir}/config.yaml"),
+            content=Template("config.yaml.j2"),
+            owner = params.storm_user,
+            group = params.user_group
+  )
+
   yaml_config( "storm.yaml",
                conf_dir = params.conf_dir,
                configurations = params.config['configurations']['storm-site'],

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/STORM/package/templates/config.yaml.j2

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nimbusHost: {{nimbus_host}}
+nimbusPort: {{nimbus_port}}
+
+# HTTP-specific options.
+http:
+
+  # The port on which the HTTP server listens for service requests.
+  port: {{rest_api_port}}
+
+  # The port on which the HTTP server listens for administrative requests.
+  adminPort: {{rest_api_admin_port}}