AMBARI-11890 [WinTP2] [Test] Cluster install via blueprint fails

Strengthened the parameter assignments originating from environment vars, to handle out-of-order installations

ambari-common/src/main/python/resource_management/libraries/functions/install_hdp_msi.py

@@ -97,7 +97,7 @@ INSTALL_MSI_CMD = 'cmd /C start /wait msiexec /qn /i  {hdp_msi_path} /lv {hdp_lo
                   'HDP_LAYOUT={hdp_layout_path} DESTROY_DATA=yes HDP_USER={hadoop_user} HDP_USER_PASSWORD={hadoop_password_arg} HDP=yes ' \
                   'KNOX=yes KNOX_MASTER_SECRET="AmbariHDP2Windows" FALCON=yes STORM=yes HBase=yes STORM=yes FLUME=yes SLIDER=yes PHOENIX=no RANGER=no'
 CREATE_SERVICE_SCRIPT = os.path.abspath("sbin\createservice.ps1")
-CREATE_SERVICE_CMD = 'cmd /C powershell -File "{script}" -username {username} -password "{password}" -servicename ' \
+CREATE_SERVICE_CMD = 'cmd /C powershell -ExecutionPolicy Bypass -File "{script}" -username {username} -password "{password}" -servicename ' \
                      '{servicename} -hdpresourcesdir "{resourcedir}" -servicecmdpath "{servicecmd}"'
 INSTALL_MARKER_OK = "msi.installed"
 INSTALL_MARKER_FAILED = "msi.failed"

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py

@@ -143,7 +143,9 @@ class DataNodeDefault(DataNode):
 
 @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
 class DataNodeWindows(DataNode):
-  pass
+  def install(self, env):
+    import install_params
+    self.install_packages(env, install_params.exclude_packages)
 
 if __name__ == "__main__":
   DataNode().execute()

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py

@@ -111,7 +111,10 @@ class HdfsClientDefault(HdfsClient):
 
 @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
 class HdfsClientWindows(HdfsClient):
-  pass
+  def install(self, env):
+    import install_params
+    self.install_packages(env, install_params.exclude_packages)
+    self.configure(env)
 
 if __name__ == "__main__":
   HdfsClient().execute()

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py

@@ -0,0 +1,39 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons import OSCheck
+
+# These parameters are supposed to be referenced at installation time, before the Hadoop environment variables have been set
+if OSCheck.is_windows_family():
+  exclude_packages = []
+else:
+  from resource_management.libraries.functions.default import default
+  from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
+  from resource_management.libraries.script.script import Script
+
+  _config = Script.get_config()
+  stack_version_unformatted = str(_config['hostLevelParams']['stack_version'])
+
+  # The logic for LZO also exists in OOZIE's params.py
+  io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
+  lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+  lzo_packages = get_lzo_packages(stack_version_unformatted)
+
+  exclude_packages = []
+  if not lzo_enabled:
+    exclude_packages += lzo_packages

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py

@@ -159,6 +159,9 @@ class JournalNodeDefault(JournalNode):
 
 @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
 class JournalNodeWindows(JournalNode):
+  def install(self, env):
+    import install_params
+    self.install_packages(env, install_params.exclude_packages)
 
   def start(self, env):
     import params

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py

@@ -260,6 +260,11 @@ class NameNodeDefault(NameNode):
 
 @OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
 class NameNodeWindows(NameNode):
+  def install(self, env):
+    import install_params
+    self.install_packages(env, install_params.exclude_packages)
+    #TODO we need this for HA because of manual steps
+    self.configure(env)
 
   def rebalancehdfs(self, env):
     from ambari_commons.os_windows import UserHelper, run_os_command_impersonated

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py

@@ -17,8 +17,10 @@ limitations under the License.
 
 """
 
-from resource_management import *
 import os
+
+#Used in subsequent imports from params
+from install_params import exclude_packages
 from status_params import *
 
 config = Script.get_config()
@@ -64,7 +66,6 @@ hdfs_user = hadoop_user
 grep_exe = "findstr"
 
 name_node_params = default("/commandParams/namenode", None)
-exclude_packages = []
 
 service_map = {
   "datanode" : datanode_win_service_name,

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_windows.py

@@ -28,14 +28,28 @@ config = Script.get_config()
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 
-hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
-hive_conf_dir = os.environ["HIVE_CONF_DIR"]
-hive_home = os.environ["HIVE_HOME"]
-hive_lib_dir = os.environ["HIVE_LIB_DIR"]
-hive_log_dir = os.environ["HIVE_LOG_DIR"]
-hive_opts = os.environ["HIVE_OPTS"]
-hcat_home = os.environ["HCAT_HOME"]
-hcat_config_dir = os.environ["WEBHCAT_CONF_DIR"]
+hdp_root = None
+hive_conf_dir = None
+hive_home = None
+hive_lib_dir = None
+hive_log_dir = None
+hive_opts = None
+hcat_home = None
+hcat_config_dir = None
+hive_bin = None
+
+try:
+  hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
+  hive_conf_dir = os.environ["HIVE_CONF_DIR"]
+  hive_home = os.environ["HIVE_HOME"]
+  hive_lib_dir = os.environ["HIVE_LIB_DIR"]
+  hive_log_dir = os.environ["HIVE_LOG_DIR"]
+  hive_opts = os.environ["HIVE_OPTS"]
+  hcat_home = os.environ["HCAT_HOME"]
+  hcat_config_dir = os.environ["WEBHCAT_CONF_DIR"]
+  hive_bin = os.path.join(hive_home, "bin")
+except:
+  pass
 
 hive_env_sh_template = config['configurations']['hive-env']['content']
 hive_warehouse_dir = config['configurations']['hive-site']['hive.metastore.warehouse.dir']
@@ -43,7 +57,6 @@ hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
 hive_user = hadoop_user
 hcat_user = hadoop_user
 
-hive_bin = os.path.join(hive_home, "bin")
 hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type']
 hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName']
 hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword']

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_windows.py

@@ -25,17 +25,32 @@ from status_params import *
 # server configurations
 config = Script.get_config()
 
-hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
-knox_home = os.environ['KNOX_HOME']
-knox_conf_dir = os.environ['KNOX_CONF_DIR']
-knox_logs_dir = os.environ['KNOX_LOG_DIR']
-knox_bin = os.path.join(knox_home, 'bin', 'gateway.exe')
-ldap_bin = os.path.join(knox_home, 'bin', 'ldap.exe')
-knox_client_bin = os.path.join(knox_home, 'bin', 'knoxcli.cmd')
-knox_data_dir = os.path.join(knox_home, 'data')
-
-knox_master_secret_path = os.path.join(knox_data_dir, 'security', 'master')
-knox_cert_store_path = os.path.join(knox_data_dir, 'security', 'keystores', 'gateway.jks')
+hdp_root = None
+knox_home = None
+knox_conf_dir = None
+knox_logs_dir = None
+knox_bin = None
+ldap_bin = None
+knox_client_bin = None
+knox_data_dir = None
+
+knox_master_secret_path = None
+knox_cert_store_path = None
+
+try:
+  hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
+  knox_home = os.environ['KNOX_HOME']
+  knox_conf_dir = os.environ['KNOX_CONF_DIR']
+  knox_logs_dir = os.environ['KNOX_LOG_DIR']
+  knox_bin = os.path.join(knox_home, 'bin', 'gateway.exe')
+  ldap_bin = os.path.join(knox_home, 'bin', 'ldap.exe')
+  knox_client_bin = os.path.join(knox_home, 'bin', 'knoxcli.cmd')
+  knox_data_dir = os.path.join(knox_home, 'data')
+
+  knox_master_secret_path = os.path.join(knox_data_dir, 'security', 'master')
+  knox_cert_store_path = os.path.join(knox_data_dir, 'security', 'keystores', 'gateway.jks')
+except:
+  pass
 
 knox_host_port = config['configurations']['gateway-site']['gateway.port']
 knox_host_name = config['clusterHostInfo']['knox_gateway_hosts'][0]

ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_windows.py

@@ -22,9 +22,17 @@ from resource_management import *
 
 # server configurations
 config = Script.get_config()
-hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
-pig_home = os.environ['PIG_HOME']
-pig_conf_dir = os.path.join(pig_home,'conf')
+
+hdp_root = None
+pig_home = None
+pig_conf_dir = None
+try:
+  hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
+  pig_home = os.environ['PIG_HOME']
+  pig_conf_dir = os.path.join(pig_home,'conf')
+except:
+  pass
+
 pig_properties = config['configurations']['pig-properties']['content']
 
 if (('pig-log4j' in config['configurations']) and ('content' in config['configurations']['pig-log4j'])):

ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params_windows.py

@@ -24,11 +24,20 @@ import os
 # server configurations
 config = Script.get_config()
 
-hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
-slider_home = os.environ['SLIDER_HOME']
-slider_bin_dir = os.path.join(slider_home, 'bin')
-slider_conf_dir = os.path.join(slider_home, 'conf')
-storm_slider_conf_dir = os.path.join(os.environ['STORM_HOME'], 'conf')
+hdp_root = None
+slider_home = None
+slider_bin_dir = None
+slider_conf_dir = None
+storm_slider_conf_dir = None
+try:
+  hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
+  slider_home = os.environ['SLIDER_HOME']
+  slider_bin_dir = os.path.join(slider_home, 'bin')
+  slider_conf_dir = os.path.join(slider_home, 'conf')
+  storm_slider_conf_dir = os.path.join(os.environ['STORM_HOME'], 'conf')
+except:
+  pass
+
 slider_home_dir = slider_home
 
 hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]

ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py

@@ -36,8 +36,11 @@ except KeyError:
 
 hdp_stack_version = ""
 
-hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], ".."))
-
+hdp_root = None
+try:
+  hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], ".."))
+except:
+  pass
 
 def refresh_tez_state_dependent_params():
   global tez_home_dir, tez_conf_dir, hdp_stack_version

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/params_windows.py

@@ -25,9 +25,15 @@ import status_params
 # server configurations
 config = Script.get_config()
 
-# notused zookeeper_home_dir = os.environ["ZOOKEEPER_HOME"]
-config_dir = os.environ["ZOOKEEPER_CONF_DIR"]
-hdp_root = os.environ["HADOOP_NODE_INSTALL_ROOT"]
+config_dir = None
+hdp_root = None
+try:
+  # not used zookeeper_home_dir = os.environ["ZOOKEEPER_HOME"]
+  config_dir = os.environ["ZOOKEEPER_CONF_DIR"]
+  hdp_root = os.environ["HADOOP_NODE_INSTALL_ROOT"]
+except:
+  pass
+
 hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
 zk_user = hadoop_user
 
@@ -59,4 +65,4 @@ zookeeper_win_service_name = status_params.zookeeper_win_service_name
 if (('zookeeper-log4j' in config['configurations']) and ('content' in config['configurations']['zookeeper-log4j'])):
   log4j_props = config['configurations']['zookeeper-log4j']['content']
 else:
-  log4j_props = None
+  log4j_props = None