AMBARI-3931. Sqoop on HDP2. Using resource management lib (Eugene Chekanskiy via dlysnichenko)

ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/metainfo.xml

@@ -16,15 +16,45 @@
    limitations under the License.
 -->
 <metainfo>
-    <user>root</user>
-    <comment>Tool for transferring bulk data between Apache Hadoop and structured data stores such as relational databases</comment>
-    <version>1.4.4.2.0.6.0</version>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>SQOOP</name>
+      <comment>Tool for transferring bulk data between Apache Hadoop and
+        structured data stores such as relational databases
+      </comment>
+      <version>1.4.4.2.0.6.0</version>
 
-    <components>
+      <components>
         <component>
-            <name>SQOOP</name>
-            <category>CLIENT</category>
+          <name>SQOOP</name>
+          <category>CLIENT</category>
+          <commandScript>
+            <script>scripts/sqoop_client.py</script>
+            <scriptType>PYTHON</scriptType>
+          </commandScript>
         </component>
-    </components>
-
+      </components>
+      <osSpecifics>
+        <osSpecific>
+          <osType>any</osType>
+          <packages>
+            <package>
+              <type>rpm</type>
+              <name>sqoop</name>
+            </package>
+            <package>
+              <type>rpm</type>
+              <name>mysql-connector-java</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
+      <commandScript>
+        <script>scripts/service_check.py</script>
+        <scriptType>PYTHON</scriptType>
+        <timeout>300</timeout>
+      </commandScript>
+    </service>
+  </services>
 </metainfo>

ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/__init__.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python2.6
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""

ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/params.py

@@ -0,0 +1,40 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+
+security_enabled = config['configurations']['global']['security_enabled']
+smokeuser = config['configurations']['global']['smokeuser']
+user_group = config['configurations']['global']['user_group']
+
+sqoop_conf_dir = "/usr/lib/sqoop/conf"
+hbase_home = "/usr"
+hive_home = "/usr"
+zoo_conf_dir = "/etc/zookeeper"
+sqoop_lib = "/usr/lib/sqoop/lib"
+sqoop_user = "sqoop"
+
+keytab_path = default("keytab_path","/etc/security/keytabs")
+
+smoke_user_keytab = "/etc/security/keytabs/smokeuser.headless.keytab" # smoke_user_keytab
+kinit_path_local = functions.get_kinit_path([default('kinit_path_local'),"/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])

ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python2.6
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+
+from resource_management import *
+
+
+class SqoopServiceCheck(Script):
+  def service_check(self, env):
+    import params
+    env.set_params(params)
+    if params.security_enabled:
+        Execute(format("{kinit_path_local}  -kt {smoke_user_keytab} {smoke_test_user}"))
+    Execute("sqoop version",
+            user = params.smokeuser,
+            logoutput = True
+    )
+
+if __name__ == "__main__":
+  SqoopServiceCheck().execute()

ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop.py

@@ -0,0 +1,54 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+import sys
+
+def sqoop(type=None):
+  import params
+  Link(params.sqoop_lib + "/mysql-connector-java.jar",
+       to = '/usr/share/java/mysql-connector-java.jar'
+  )
+  Directory(params.sqoop_conf_dir,
+            owner = params.sqoop_user,
+            group = params.user_group
+  )
+  sqoop_TemplateConfig("sqoop-env.sh")
+  File (params.sqoop_conf_dir + "/sqoop-env-template.sh",
+          owner = params.sqoop_user,
+          group = params.user_group
+  )
+  File (params.sqoop_conf_dir + "/sqoop-site-template.xml",
+         owner = params.sqoop_user,
+         group = params.user_group
+  )
+  File (params.sqoop_conf_dir + "/sqoop-site.xml",
+         owner = params.sqoop_user,
+         group = params.user_group
+  )
+  pass
+
+def sqoop_TemplateConfig(name, tag=None):
+  import params
+  TemplateConfig( format("{sqoop_conf_dir}/{name}"),
+                  owner = params.sqoop_user,
+                  template_tag = tag
+  )

ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop_client.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python2.6
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import sys
+from resource_management import *
+
+from sqoop import sqoop
+
+
+class SqoopClient(Script):
+  def install(self, env):
+    self.install_packages(env)
+    self.configure(env)
+
+  def configure(self, env):
+    import params
+    env.set_params(params)
+    sqoop(type='client')
+
+if __name__ == "__main__":
+  SqoopClient().execute()

ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/templates/sqoop-env.sh.j2

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# included in all the hadoop scripts with source command
+# should not be executable directly
+# also should not be passed any arguments, since we need original $*
+
+# Set Hadoop-specific environment variables here.
+
+#Set path to where bin/hadoop is available
+#Set path to where bin/hadoop is available
+export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop}
+
+#set the path to where bin/hbase is available
+export HBASE_HOME=${HBASE_HOME:-/usr/lib/hbase}
+
+#Set the path to where bin/hive is available
+export HIVE_HOME=${HIVE_HOME:-/usr/lib/hive}
+
+#Set the path for where zookeper config dir is
+export ZOOCFGDIR=${ZOOCFGDIR:-/etc/zookeeper/conf}
+
+# add libthrift in hive to sqoop class path first so hive imports work
+export SQOOP_USER_CLASSPATH="`ls ${HIVE_HOME}/lib/libthrift-*.jar 2> /dev/null`:${SQOOP_USER_CLASSPATH}"