AMBARI-8366 - Sqoop Configs fail to load on Services page on WINHDP (Eugene Chekanskiy via abaranchuk)

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/FALCON/package/scripts/falcon_client.py

@@ -24,7 +24,7 @@ class FalconClient(Script):
     import params
     if params.falcon_home is None:
       self.install_packages(env)
-      self.configure(env)
+    self.configure(env)
 
   def configure(self, env):
     import params

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/HDFS/package/scripts/hdfs.py

@@ -20,6 +20,7 @@ Ambari Agent
 """
 
 from resource_management import *
+import os
 
 def hdfs(component=None):
   import params
@@ -50,12 +51,12 @@ def hdfs(component=None):
             mode="f",
             configuration_attributes=params.config['configuration_attributes']['hdfs-site']
   )
-  File(format("{params.hadoop_conf_dir}/hadoop-metrics2.properties"),
+  File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"),
        content=Template("hadoop-metrics2.properties.j2"),
        owner=params.hdfs_user,
        mode="f"
   )
-  File(format("{params.hbase_conf_dir}/hadoop-metrics2-hbase.properties"),
+  File(os.path.join(params.hbase_conf_dir, "hadoop-metrics2-hbase.properties"),
        content=Template("hadoop-metrics2.properties.j2"),
        owner=params.hdfs_user,
        mode="f"

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/OOZIE/package/scripts/oozie.py

@@ -20,6 +20,7 @@ Ambari Agent
 """
 
 from resource_management import *
+import os
 
 def oozie():
   import params
@@ -31,7 +32,7 @@ def oozie():
             configuration_attributes=params.config['configuration_attributes']['oozie-site']
   )
 
-  File(format("{oozie_conf_dir}/oozie-env.cmd"),
+  File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"),
     owner=params.oozie_user,
     content=InlineTemplate(params.oozie_env_cmd_template)
   )

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py

@@ -24,14 +24,14 @@ from resource_management import *
 
 def pig():
   import params
-  File(format("{params.pig_conf_dir}/pig.properties"),
+  File(os.path.join(params.pig_conf_dir, "pig.properties"),
        mode="f",
        owner=params.pig_user,
        content=params.pig_properties
   )
 
   if (params.log4j_props != None):
-    File(format("{params.pig_conf_dir}/log4j.properties"),
+    File(os.path.join(params.pig_conf_dir, "log4j.properties"),
          mode='f',
          owner=params.pig_user,
          content=params.log4j_props
@@ -44,6 +44,6 @@ def pig_TemplateConfig(name):
     name = [name]
 
   for x in name:
-    TemplateConfig( format("{pig_conf_dir}/{x}"),
+    TemplateConfig(os.path.join(params.pig_conf_dir,x),
         owner = params.hdfs_user
     )

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/configuration/sqoop-site.xml

@@ -1,156 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied.  See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<!-- Put Sqoop-specific properties in this file. -->
-
-<configuration>
-
-  <!--
-    Set the value of this property to explicitly enable third-party
-    ManagerFactory plugins.
-
-    If this is not used, you can alternately specify a set of ManagerFactories
-    in the $SQOOP_CONF_DIR/managers.d/ subdirectory.  Each file should contain
-    one or more lines like:
-      manager.class.name[=/path/to/containing.jar]
-
-    Files will be consulted in lexicographical order only if this property
-    is unset.
-  -->
-  <!--
-  <property>
-    <name>sqoop.connection.factories</name>
-    <value>com.cloudera.sqoop.manager.DefaultManagerFactory</value>
-    <description>A comma-delimited list of ManagerFactory implementations
-      which are consulted, in order, to instantiate ConnManager instances
-      used to drive connections to databases.
-    </description>
-  </property>
-  -->
-
-  <!--
-    Set the value of this property to enable third-party tools.
-
-    If this is not used, you can alternately specify a set of ToolPlugins
-    in the $SQOOP_CONF_DIR/tools.d/ subdirectory.  Each file should contain
-    one or more lines like:
-      plugin.class.name[=/path/to/containing.jar]
-
-    Files will be consulted in lexicographical order only if this property
-    is unset.
-  -->
-  <!--
-  <property>
-    <name>sqoop.tool.plugins</name>
-    <value></value>
-    <description>A comma-delimited list of ToolPlugin implementations
-      which are consulted, in order, to register SqoopTool instances which
-      allow third-party tools to be used.
-    </description>
-  </property>
-  -->
-
-  <!--
-    By default, the Sqoop metastore will auto-connect to a local embedded
-    database stored in ~/.sqoop/. To disable metastore auto-connect, uncomment
-    this next property.
-  -->
-  <!--
-  <property>
-    <name>sqoop.metastore.client.enable.autoconnect</name>
-    <value>false</value>
-    <description>If true, Sqoop will connect to a local metastore
-      for job management when no other metastore arguments are
-      provided.
-    </description>
-  </property>
-  -->
-
-  <!--
-    The auto-connect metastore is stored in ~/.sqoop/. Uncomment
-    these next arguments to control the auto-connect process with
-    greater precision.
-  -->
-  <!--
-  <property>
-    <name>sqoop.metastore.client.autoconnect.url</name>
-    <value>jdbc:hsqldb:file:/tmp/sqoop-meta/meta.db;shutdown=true</value>
-    <description>The connect string to use when connecting to a
-      job-management metastore. If unspecified, uses ~/.sqoop/.
-      You can specify a different path here.
-    </description>
-  </property>
-  <property>
-    <name>sqoop.metastore.client.autoconnect.username</name>
-    <value>SA</value>
-    <description>The username to bind to the metastore.
-    </description>
-  </property>
-  <property>
-    <name>sqoop.metastore.client.autoconnect.password</name>
-    <value></value>
-    <description>The password to bind to the metastore.
-    </description>
-  </property>
-  -->
-
-  <!--
-    For security reasons, by default your database password will not be stored in
-    the Sqoop metastore. When executing a saved job, you will need to
-    reenter the database password. Uncomment this setting to enable saved
-    password storage. (INSECURE!)
-  -->
-  <!--
-  <property>
-    <name>sqoop.metastore.client.record.password</name>
-    <value>true</value>
-    <description>If true, allow saved passwords in the metastore.
-    </description>
-  </property>
-  -->
-
-
-  <!--
-    SERVER CONFIGURATION: If you plan to run a Sqoop metastore on this machine,
-    you should uncomment and set these parameters appropriately.
-
-    You should then configure clients with:
-       sqoop.metastore.client.autoconnect.url =
-       jdbc:hsqldb:hsql://&lt;server-name&gt;:&lt;port&gt;/sqoop
-  -->
-  <!--
-  <property>
-    <name>sqoop.metastore.server.location</name>
-    <value>/tmp/sqoop-metastore/shared.db</value>
-    <description>Path to the shared metastore database files.
-    If this is not set, it will be placed in ~/.sqoop/.
-    </description>
-  </property>
-
-  <property>
-    <name>sqoop.metastore.server.port</name>
-    <value>16000</value>
-    <description>Port that this metastore should listen on.
-    </description>
-  </property>
-  -->
-
-</configuration>

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/params.py

@@ -26,7 +26,7 @@ sqoop_home_dir = None
 sqoop_conf_dir = "conf"
 
 hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], ".."))
-
+sqoop_env_cmd_template = config['configurations']['sqoop-env']['content']
 if os.environ.has_key("SQOOP_HOME"):
   sqoop_home_dir = os.environ["SQOOP_HOME"]
   sqoop_conf_dir = os.path.join(sqoop_home_dir, "conf")

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop.py

@@ -18,13 +18,10 @@ limitations under the License.
 """
 
 from resource_management import *
-import sys
+import os
 
 def sqoop():
   import params
-  XmlConfig("sqoop-site.xml",
-              conf_dir=params.sqoop_conf_dir,
-              configurations=params.config['configurations']['sqoop-site'],
-              owner=params.sqoop_user,
-              mode="f"
+  File(os.path.join(params.sqoop_conf_dir, "sqoop-env.cmd"),
+       content=InlineTemplate(params.sqoop_env_cmd_template)
   )

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop_client.py

@@ -20,14 +20,14 @@ limitations under the License.
 
 import sys
 from resource_management import *
-
-
+from sqoop import sqoop
 
 class SqoopClient(Script):
   def install(self, env):
-      import params
-      if params.sqoop_home_dir is None:
-          self.install_packages(env)
+    import params
+    if params.sqoop_home_dir is None:
+      self.install_packages(env)
+    self.configure(env)
 
   def configure(self, env):
     import params

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/nimbus.py

@@ -28,7 +28,7 @@ class Nimbus(Script):
   def install(self, env):
     if not check_windows_service_exists(service_mapping.nimbus_win_service_name):
       self.install_packages(env)
-      self.configure(env)
+    self.configure(env)
 
   def start(self, env):
     self.configure(env)

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/supervisor.py

@@ -28,7 +28,7 @@ class Supervisor(Script):
   def install(self, env):
     if not check_windows_service_exists(service_mapping.supervisor_win_service_name):
       self.install_packages(env)
-      self.configure(env)
+    self.configure(env)
 
   def start(self, env):
     self.configure(env)

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/yaml_config.py

@@ -20,6 +20,7 @@ limitations under the License.
 
 
 from resource_management import *
+import os
 
 def escape_yaml_propetry(value):
   unquouted = False
@@ -57,10 +58,11 @@ def yaml_config(
   owner = None,
   group = None
 ):
+    import params
     config_content = source.InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_propetry(value) }}
 {% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_propetry])
 
-    File (format("{conf_dir}/{filename}"),
+    File (os.path.join(params.conf_dir, filename),
       content = config_content,
       owner = owner,
       mode = "f"

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py

@@ -27,6 +27,7 @@ class TezClient(Script):
     import params
     if params.tez_home_dir is None:
       self.install_packages(env)
+    self.configure(env)
 
   def configure(self, env):
     import params

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/package/scripts/zookeeper.py

@@ -35,12 +35,12 @@ def zookeeper(type = None):
             recursive=True
   )
   if (params.log4j_props != None):
-    File(format("{params.zookeeper_conf_dir}/log4j.properties"),
+    File(os.path.join(params.zookeeper_conf_dir, "log4j.properties"),
          mode="f",
          owner=params.zk_user,
          content=params.log4j_props
     )
-  elif (os.path.exists(format("{params.zookeeper_conf_dir}/log4j.properties"))):
+  elif (os.path.exists(os.path.join(params.zookeeper_conf_dir, "log4j.properties"))):
     File(format("{params.zookeeper_conf_dir}/log4j.properties"),
          mode="f",
          owner=params.zk_user
@@ -56,7 +56,7 @@ def zookeeper(type = None):
 def configFile(name, template_name=None):
   import params
 
-  File(format("{params.zookeeper_conf_dir}/{name}"),
+  File(os.path.join(params.zookeeper_conf_dir, name),
        content=Template(template_name),
        owner=params.zk_user,
        mode="f"

ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/package/scripts/zookeeper_client.py

@@ -29,6 +29,7 @@ class ZookeeperClient(Script):
     # client checks env var to determine if it is installed
     if not os.environ.has_key("ZOOKEEPER_HOME"):
       self.install_packages(env)
+    self.configure(env)
 
   def configure(self, env):
     import params