AMBARI-9269 - Add FLUME service to HDPWIN 2.1 and HDPWIN 2.2

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume.py

@@ -23,7 +23,52 @@ import os
 from resource_management import *
 from resource_management.libraries.functions.flume_agent_helper import is_flume_process_live
 from resource_management.libraries.functions.flume_agent_helper import find_expected_agent_names
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def flume(action = None):
+  import params
+
+  if action == 'config':
+    # remove previously defined meta's
+    for n in find_expected_agent_names(params.flume_conf_dir):
+      os.unlink(os.path.join(params.flume_conf_dir, n, 'ambari-meta.json'))
+
+    flume_agents = {}
+    if params.flume_conf_content is not None:
+      flume_agents = build_flume_topology(params.flume_conf_content)
+
+    for agent in flume_agents.keys():
+      flume_agent_conf_dir = os.path.join(params.flume_conf_dir, agent)
+      flume_agent_conf_file = os.path.join(flume_agent_conf_dir, 'flume.conf')
+      flume_agent_meta_file = os.path.join(flume_agent_conf_dir, 'ambari-meta.json')
+      flume_agent_log4j_file = os.path.join(flume_agent_conf_dir, 'log4j.properties')
+      flume_agent_env_file = os.path.join(flume_agent_conf_dir, 'flume-env.ps1')
+
+      Directory(flume_agent_conf_dir)
+
+      PropertiesFile(flume_agent_conf_file,
+                     properties=flume_agents[agent])
+
+      File(flume_agent_log4j_file,
+           content=Template('log4j.properties.j2', agent_name = agent))
+
+      File(flume_agent_meta_file,
+           content = json.dumps(ambari_meta(agent, flume_agents[agent])))
+
+      File(flume_agent_env_file,
+           owner=params.flume_user,
+           content=InlineTemplate(params.flume_env_sh_template)
+      )
+
+      if params.has_metric_collector:
+        File(os.path.join(flume_agent_conf_dir, "flume-metrics2.properties"),
+             owner=params.flume_user,
+             content=Template("flume-metrics2.properties.j2")
+        )
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
 def flume(action = None):
   import params
 

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_check.py

@@ -18,13 +18,22 @@ limitations under the License.
 """
 
 from resource_management import *
-
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 
 class FlumeServiceCheck(Script):
 
+  @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
   def service_check(self, env):
     import params
+    env.set_params(params)
+    smoke_cmd = os.path.join(params.hdp_root,"Run-SmokeTests.cmd")
+    service = "FLUME"
+    Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True, user=params.hdfs_user)
 
+  @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+  def service_check(self, env):
+    import params
     env.set_params(params)
     if params.security_enabled:
       principal_replaced = params.http_principal.replace("_HOST", params.hostname)

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py

@@ -26,48 +26,65 @@ from resource_management import *
 from resource_management.libraries.functions.flume_agent_helper import find_expected_agent_names
 from resource_management.libraries.functions.flume_agent_helper import get_flume_status
 from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
+import service_mapping
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 
 class FlumeHandler(Script):
 
+  @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
   def get_stack_to_component(self):
     return {"HDP": "flume-server"}
 
+  @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
   def install(self, env):
     import params
-
     self.install_packages(env)
     env.set_params(params)
 
+  @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+  def install(self, env):
+    if not check_windows_service_exists(service_mapping.flume_win_service_name):
+      self.install_packages(env)
+    self.configure(env)
+
+  @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
   def start(self, env, rolling_restart=False):
     import params
-
     env.set_params(params)
     self.configure(env)
-
     flume(action='start')
 
-  def stop(self, env, rolling_restart=False):
+  @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+  def start(self, env):
     import params
-
     env.set_params(params)
+    self.configure(env)
+    Service(service_mapping.flume_win_service_name, action="start")
+    # flume(action='start')
 
+  @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+  def stop(self, env, rolling_restart=False):
+    import params
+    env.set_params(params)
     flume(action='stop')
-
     if rolling_restart:
       flume_upgrade.post_stop_backup()
 
-  def configure(self, env):
+  @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+  def stop(self, env):
     import params
+    Service(service_mapping.flume_win_service_name, action="stop")
 
+  def configure(self, env):
+    import params
     env.set_params(params)
-
     flume(action='config')
 
+  @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
   def status(self, env):
     import params
     env.set_params(params)
-
-
     processes = get_flume_status(params.flume_conf_dir, params.flume_run_dir)
     expected_agents = find_expected_agent_names(params.flume_conf_dir)
 
@@ -85,6 +102,12 @@ class FlumeHandler(Script):
     elif len(expected_agents) == 0 and 'INSTALLED' == get_desired_state():
       raise ComponentIsNotRunning()
 
+  @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+  def status(self, env):
+    import params
+    check_windows_service_status(service_mapping.flume_win_service_name)
+
+  @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
   def pre_rolling_restart(self, env):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py

@@ -20,6 +20,12 @@ limitations under the License.
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management.libraries.functions.default import default
 from resource_management import *
+from ambari_commons import OSCheck
+
+if OSCheck.is_windows_family():
+  from params_windows import *
+else:
+  from params_linux import *
 
 config = Script.get_config()
 
@@ -46,15 +52,9 @@ else:
   flume_hive_home = '/usr/lib/hive'
   flume_hcat_home = '/usr/lib/hive-hcatalog'
 
-flume_conf_dir = '/etc/flume/conf'
 java_home = config['hostLevelParams']['java_home']
 flume_log_dir = '/var/log/flume'
 flume_run_dir = '/var/run/flume'
-flume_user = 'flume'
-flume_group = 'flume'
-
-if 'flume-env' in config['configurations'] and 'flume_user' in config['configurations']['flume-env']:
-  flume_user = config['configurations']['flume-env']['flume_user']
 
 if (('flume-conf' in config['configurations']) and('content' in config['configurations']['flume-conf'])):
   flume_conf_content = config['configurations']['flume-conf']['content']

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params_linux.py

@@ -0,0 +1,32 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
+from resource_management import *
+from ambari_commons import OSCheck
+
+# server configurations
+config = Script.get_config()
+
+flume_conf_dir = '/etc/flume/conf'
+flume_user = 'flume'
+flume_group = 'flume'
+if 'flume-env' in config['configurations'] and 'flume_user' in config['configurations']['flume-env']:
+  flume_user = config['configurations']['flume-env']['flume_user']

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params_windows.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+import os
+
+# server configurations
+config = Script.get_config()
+
+hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
+flume_home = os.environ['FLUME_HOME']
+flume_conf_dir = os.path.join(flume_home, 'conf')
+flume_user = 'hadoop'
+hdfs_user = "hadoop"

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/service_mapping.py

@@ -0,0 +1,21 @@
+# !/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+flume_win_service_name = "flumeagent"

ambari-server/src/main/resources/stacks/HDPWIN/2.1/role_command_order.json

@@ -13,12 +13,8 @@
     "HIVE_SERVER-START": ["TASKTRACKER-START", "MYSQL_SERVER-START", "NAMENODE-START", "DATANODE-START", "SECONDARY_NAMENODE-START"],
     "HUE_SERVER-START": ["HIVE_SERVER-START", "HCAT-START", "OOZIE_SERVER-START"],
     "FLUME_HANDLER-START": ["OOZIE_SERVER-START"],
-    "NAGIOS_SERVER-START": ["HBASE_MASTER-START", "HBASE_REGIONSERVER-START",
-        "GANGLIA_SERVER-START", "GANGLIA_MONITOR-START", "HCAT-START",
-        "HIVE_SERVER-START", "HIVE_METASTORE-START", "HUE_SERVER-START",
-        "JOBTRACKER-START", "TASKTRACKER-START", "ZOOKEEPER_SERVER-START",
-        "MYSQL_SERVER-START", "OOZIE_SERVER-START", "PIG-START", "SQOOP-START",
-        "WEBHCAT_SERVER-START", "FLUME_HANDLER-START"],
+    "FLUME_SERVICE_CHECK-SERVICE_CHECK": ["FLUME_HANDLER-START"],
+    "SLIDER_SERVICE_CHECK-SERVICE_CHECK" : ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
     "MAPREDUCE_SERVICE_CHECK-SERVICE_CHECK": ["JOBTRACKER-START", "TASKTRACKER-START"],
     "OOZIE_SERVICE_CHECK-SERVICE_CHECK": ["OOZIE_SERVER-START", "MAPREDUCE2_SERVICE_CHECK-SERVICE_CHECK"],
     "HBASE_SERVICE_CHECK-SERVICE_CHECK": ["HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
@@ -45,10 +41,7 @@
     "OOZIE_CLIENT-UPGRADE" : ["OOZIE_SERVER-UPGRADE"],
     "WEBHCAT_SERVER-UPGRADE" : ["OOZIE_CLIENT-UPGRADE"],
     "PIG-UPGRADE" : ["WEBHCAT_SERVER-UPGRADE"],
-    "SQOOP-UPGRADE" : ["PIG-UPGRADE"],
-    "NAGIOS_SERVER-UPGRADE" : ["SQOOP-UPGRADE"],
-    "GANGLIA_SERVER-UPGRADE" : ["NAGIOS_SERVER-UPGRADE"],
-    "GANGLIA_MONITOR-UPGRADE" : ["GANGLIA_SERVER-UPGRADE"]
+    "SQOOP-UPGRADE" : ["PIG-UPGRADE"]
   },
   "_comment" : "GLUSTERFS-specific dependencies",
   "optional_glusterfs": {
@@ -70,8 +63,6 @@
     "HIVE_METASTORE-START": ["NAMENODE-START", "DATANODE-START", "SECONDARY_NAMENODE-START"],
     "HIVE_SERVER-START": ["NAMENODE-START", "DATANODE-START", "SECONDARY_NAMENODE-START"],
     "WEBHCAT_SERVER-START": ["DATANODE-START"],
-    "NAGIOS_SERVER-START": ["NAMENODE-START", "SECONDARY_NAMENODE-START",
-        "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START", "HISTORYSERVER-START"],
     "HDFS_SERVICE_CHECK-SERVICE_CHECK": ["NAMENODE-START", "DATANODE-START",
         "SECONDARY_NAMENODE-START"],
     "MAPREDUCE2_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START",
@@ -91,8 +82,7 @@
   "_comment" : "Dependencies that are used in HA NameNode cluster",
   "namenode_optional_ha": {
     "NAMENODE-START": ["ZKFC-START", "JOURNALNODE-START", "ZOOKEEPER_SERVER-START"],
-    "ZKFC-START": ["ZOOKEEPER_SERVER-START"],
-    "NAGIOS_SERVER-START": ["ZKFC-START", "JOURNALNODE-START"]
+    "ZKFC-START": ["ZOOKEEPER_SERVER-START"]
   },
   "_comment" : "Dependencies that are used in ResourceManager HA cluster",
   "resourcemanager_optional_ha" : {

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/FLUME/configuration/flume-conf.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="false">
+  <property>
+    <name>content</name>
+    <description>Flume agent configurations. 
+    Specifying the configuration here applies to all hosts in this configuration-group. Please use host specific configuration-groups to provide different configurations on different hosts. 
+    For configuring multiple agents on the same host, provide the combined agent configurations here. Each agent should have a different name.</description>
+    <value>
+# Flume agent config
+    </value>
+  </property>
+</configuration>

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/FLUME/configuration/flume-env.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <!-- flume-env.ps1 -->
+  <property>
+    <name>content</name>
+    <description>This is the jinja template for flume-env.ps1 file</description>
+    <value>
+      # Licensed to the Apache Software Foundation (ASF) under one
+      # or more contributor license agreements.  See the NOTICE file
+      # distributed with this work for additional information
+      # regarding copyright ownership.  The ASF licenses this file
+      # to you under the Apache License, Version 2.0 (the
+      # "License"); you may not use this file except in compliance
+      # with the License.  You may obtain a copy of the License at
+      #
+      #     http://www.apache.org/licenses/LICENSE-2.0
+      #
+      # Unless required by applicable law or agreed to in writing, software
+      # distributed under the License is distributed on an "AS IS" BASIS,
+      # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+      # See the License for the specific language governing permissions and
+      # limitations under the License.
+
+
+      # Give Flume more memory and pre-allocate, enable remote monitoring via JMX
+      $JAVA_OPTS="-Xms100m -Xmx2000m -Dcom.sun.management.jmxremote"
+
+      # Foll. classpath will be included in Flume's classpath.
+      # Note that the Flume conf directory is always included in the classpath.
+      $FLUME_CLASSPATH=""   # Example:  "path1;path2;path3"
+
+      # $HIVE_HOME=
+      # $HCAT_HOME=
+    </value>
+  </property>
+</configuration>

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/FLUME/metainfo.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>FLUME</name>
+      <extends>common-services/FLUME/1.4.0.2.0</extends>
+    </service>
+  </services>
+</metainfo>

ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FLUME/metainfo.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>FLUME</name>
+      <version>1.5.1.2.2.0.0-2041</version>
+    </service>
+  </services>
+</metainfo>