AMBARI-7551 Create mahout definition for bigtop (rmeneses via jaoki)

ambari-server/src/main/resources/stacks/BIGTOP/0.8/role_command_order.json

@@ -26,6 +26,7 @@
     "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START"],
     "HCAT_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START"],
     "PIG_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
+    "MAHOUT_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
     "SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
     "ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
     "ZOOKEEPER_QUORUM_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
@@ -55,6 +56,7 @@
     "YARN_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
     "RESOURCEMANAGER_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START"],
     "PIG_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START", "NODEMANAGER-START"],
+    "MAHOUT_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START", "NODEMANAGER-START"],
     "NAMENODE-STOP": ["RESOURCEMANAGER-STOP", "NODEMANAGER-STOP",
         "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"],
     "DATANODE-STOP": ["RESOURCEMANAGER-STOP", "NODEMANAGER-STOP",

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/metainfo.xml

@@ -0,0 +1,66 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>MAHOUT</name>
+      <displayName>Mahout</displayName>
+      <comment>The Apache Mahout project's goal is to build a scalable machine learning library</comment>
+      <version>0.9.666</version>
+      <components>
+        <component>
+          <name>MAHOUT</name>
+          <displayName>Mahout Client</displayName>
+          <category>CLIENT</category>
+          <cardinality>0+</cardinality>
+          <commandScript>
+            <script>scripts/mahout_client.py</script>
+            <scriptType>PYTHON</scriptType>
+            <timeout>600</timeout>
+          </commandScript>
+        </component>
+      </components>
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>any</osFamily>
+          <packages>
+            <package>
+              <name>mahout</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
+
+      <commandScript>
+        <script>scripts/service_check.py</script>
+        <scriptType>PYTHON</scriptType>
+        <timeout>600</timeout>
+      </commandScript>
+
+      <requiredServices>
+        <service>YARN</service>
+      </requiredServices>
+
+      <configuration-dependencies>
+        <config-type>global</config-type>
+      </configuration-dependencies>
+
+    </service>
+  </services>
+</metainfo>

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout.py

@@ -0,0 +1,66 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import os
+
+from resource_management import *
+
+def mahout():
+  import params
+
+  Directory( params.mahout_conf_dir,
+    owner = params.hdfs_user,
+    group = params.user_group
+  )
+
+  mahout_TemplateConfig( ['mahout-env.sh'])
+
+  # mahout_properties is always set to a default even if it's not in the payload
+  File(format("{mahout_conf_dir}/mahout.properties"),
+              mode=0644,
+              group=params.user_group,
+              owner=params.hdfs_user,
+              content=params.mahout_properties
+  )
+
+  if params.log4j_props:
+    File(format("{mahout_conf_dir}/log4j.properties"),
+      mode=0644,
+      group=params.user_group,
+      owner=params.hdfs_user,
+      content=params.log4j_props
+    )
+  elif (os.path.exists(format("{mahout_conf_dir}/log4j.properties"))):
+    File(format("{mahout_conf_dir}/log4j.properties"),
+      mode=0644,
+      group=params.user_group,
+      owner=params.hdfs_user
+    )
+
+def mahout_TemplateConfig(name):
+  import params
+
+  if not isinstance(name, list):
+    name = [name]
+
+  for x in name:
+    TemplateConfig( format("{mahout_conf_dir}/{x}"),
+        owner = params.hdfs_user
+    )

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout_client.py

@@ -0,0 +1,36 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+Ambari Agent
+"""
+import sys
+from resource_management import *
+from mahout import mahout
+
+class MahoutClient(Script):
+
+  def install(self, env):
+    self.install_packages(env)
+    self.configure(env)
+
+  def configure(self, env):
+    import params
+    env.set_params(params)
+    mahout()
+
+  def status(self, env):
+    raise ClientComponentHasNoStatus()
+
+if __name__ == "__main__":
+  MahoutClient().execute()

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/params.py

@@ -0,0 +1,55 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+
+# server configurations
+config = Script.get_config()
+
+hadoop_log_dir = "/var/log/hadoop"
+mahout_conf_dir = "/etc/mahout/conf"
+hadoop_conf_dir = "/etc/hadoop/conf"
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+user_group = config['configurations']['cluster-env']['user_group']
+_authentication = config['configurations']['core-site']['hadoop.security.authentication']
+security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+
+# not supporting 32 bit jdk.
+java64_home = config['hostLevelParams']['java_home']
+hadoop_home = "/usr/lib/hadoop/"
+
+# mahout.properties - if not in the JSON command, then we need to esnure some 
+# basic properties are set; this is a safety mechanism
+if (('mahout-properties' in config['configurations']) and ('mahout-content' in config['configurations']['mahout-properties'])):
+  mahout_properties = config['configurations']['mahout-properties']['mahout-content']
+else:
+  mahout_properties = "mahout.location.check.strict=false"
+
+# log4j.properties
+if (('mahout-log4j' in config['configurations']) and ('content' in config['configurations']['mahout-log4j'])):
+  log4j_props = config['configurations']['mahout-log4j']['content']
+else:
+  log4j_props = None

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/service_check.py

@@ -0,0 +1,92 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import os
+from resource_management import *
+
+class MahoutServiceCheck(Script):
+  def service_check(self, env):
+    import params
+    env.set_params(params)
+
+    # prepare the input file content
+    input_file = os.path.join("/tmp", "mahout_input.csv")
+    input_file_content = """\
+1,101,5.0
+1,102,5.0
+1,103,2.5
+2,101,2.0
+2,102,2.5
+2,103,5.0
+2,104,2.0
+3,101,2.5
+3,104,4.0
+3,105,4.5
+3,107,5.0"""
+
+    File(input_file, content=input_file_content, mode=644)
+
+    # create the log dir for the smoke user
+    Directory(os.path.join(params.hadoop_log_dir, params.smokeuser),
+	            owner = params.smokeuser,
+              group = params.user_group,
+              mode = 755)
+
+    # transfer the input file to hdfs
+    recommenderdata_dir = "recommenderdata"
+    recommenderoutput_dir = "recommenderoutput"
+    cleanup_cmd = format("fs -rm -r {recommenderdata_dir} {recommenderoutput_dir} temp")
+    #cleanup put below to handle retries; if retrying there wil be a stale file that needs cleanup; exit code is fn of second command
+    create_file_cmd = format("{cleanup_cmd}; hadoop fs -put {input_file} {recommenderdata_dir}") #TODO: inconsistent that second command needs hadoop
+
+    test_cmd_cat = "mahout cat /etc/passwd"
+    test_cmd_recommendation = format("mahout recommenditembased --input {recommenderdata_dir} --output {recommenderoutput_dir} -s SIMILARITY_COOCCURRENCE")
+
+    ExecuteHadoop(create_file_cmd,
+      tries     = 3,
+      try_sleep = 5,
+      user      = params.smokeuser,
+      conf_dir = params.hadoop_conf_dir,
+      # for kinit run
+      keytab = params.smoke_user_keytab,
+      security_enabled = params.security_enabled,
+      kinit_path_local = params.kinit_path_local,
+      logoutput = True
+    )
+
+    Execute(test_cmd_cat,
+      tries     = 3,
+      try_sleep = 5,
+      path      = '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+      user      = params.smokeuser,
+      logoutput = True
+    )
+
+    Execute(test_cmd_recommendation,
+      tries     = 3,
+      try_sleep = 5,
+      path      = '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+      user      = params.smokeuser,
+      logoutput = True,
+    )
+
+if __name__ == "__main__":
+  MahoutServiceCheck().execute()

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/templates/mahout-env.sh.j2

@@ -0,0 +1,34 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+JAVA_HOME={{java64_home}}
+HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}