AMBARI-9581. curl unable to connect to Ambari when SSLv3 and TLSv1 is disabled (aonishuk)

ambari-agent/src/main/python/ambari_agent/AlertSchedulerHandler.py

@@ -53,7 +53,7 @@ class AlertSchedulerHandler():
   }
 
 
-  def __init__(self, cachedir, stacks_dir, common_services_dir, host_scripts_dir, in_minutes=True):
+  def __init__(self, cachedir, stacks_dir, common_services_dir, host_scripts_dir, config, in_minutes=True):
     self.cachedir = cachedir
     self.stacks_dir = stacks_dir
     self.common_services_dir = common_services_dir
@@ -69,6 +69,7 @@ class AlertSchedulerHandler():
     self._collector = AlertCollector()
     self.__scheduler = Scheduler(AlertSchedulerHandler.APS_CONFIG)
     self.__in_minutes = in_minutes
+    self.config = config
     self.__config_maps = {}
 
 
@@ -281,7 +282,7 @@ class AlertSchedulerHandler():
       source['stacks_directory'] = self.stacks_dir
       source['common_services_directory'] = self.common_services_dir
       source['host_scripts_directory'] = self.host_scripts_dir
-      alert = ScriptAlert(json_definition, source)
+      alert = ScriptAlert(json_definition, source, self.config)
     elif source_type == AlertSchedulerHandler.TYPE_WEB:
       alert = WebAlert(json_definition, source)
 

ambari-agent/src/main/python/ambari_agent/Controller.py

@@ -91,7 +91,7 @@ class Controller(threading.Thread):
     alerts_cache_dir = os.path.join(cache_dir, 'alerts')
     
     self.alert_scheduler_handler = AlertSchedulerHandler(alerts_cache_dir, 
-        stacks_cache_dir, common_services_cache_dir, host_scripts_cache_dir)
+        stacks_cache_dir, common_services_cache_dir, host_scripts_cache_dir, config)
 
 
   def __del__(self):

ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py

@@ -29,7 +29,7 @@ from symbol import parameters
 logger = logging.getLogger()
 
 class ScriptAlert(BaseAlert):
-  def __init__(self, alert_meta, alert_source_meta):
+  def __init__(self, alert_meta, alert_source_meta, config):
     """ ScriptAlert reporting structure is output from the script itself """
     
     alert_source_meta['reporting'] = {
@@ -41,6 +41,7 @@ class ScriptAlert(BaseAlert):
     
     super(ScriptAlert, self).__init__(alert_meta, alert_source_meta)
     
+    self.config = config
     self.path = None
     self.stacks_dir = None
     self.common_services_dir = None
@@ -90,7 +91,7 @@ class ScriptAlert(BaseAlert):
       matchObj = re.match( r'((.*)services\/(.*)\/package\/)', self.path_to_script)
       if matchObj:
         basedir = matchObj.group(1)
-        with Environment(basedir) as env:
+        with Environment(basedir, tmp_dir=self.config.get.get('agent', 'tmp_dir')) as env:
           return cmd_module.execute(parameters, self.host_name)
       else:
         return cmd_module.execute(parameters, self.host_name)

ambari-common/src/main/python/resource_management/core/environment.py

@@ -39,24 +39,25 @@ from resource_management.core.logger import Logger
 class Environment(object):
   _instances = []
 
-  def __init__(self, basedir=None, test_mode=False, logging_level=logging.INFO):
+  def __init__(self, basedir=None, tmp_dir=None, test_mode=False, logging_level=logging.INFO):
     """
     @param basedir: basedir/files, basedir/templates are the places where templates / static files
     are looked up
     @param test_mode: if this is enabled, resources won't be executed until manualy running env.run().
-    """
-    self.reset(basedir, test_mode)
+    """   
+    self.reset(basedir, test_mode, tmp_dir)
     
     if not Logger.logger:
       Logger.initialize_logger(logging_level)
 
-  def reset(self, basedir, test_mode):
+  def reset(self, basedir, test_mode, tmp_dir):
     self.system = System.get_instance()
     self.config = AttributeDictionary()
     self.resources = {}
     self.resource_list = []
     self.delayed_actions = set()
     self.test_mode = test_mode
+    self.tmp_dir = tmp_dir
     self.update_config({
       # current time
       'date': datetime.now(),

ambari-common/src/main/python/resource_management/core/source.py

@@ -22,12 +22,14 @@ Ambari Agent
 
 from __future__ import with_statement
 from resource_management.core.environment import Environment
+from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
 from resource_management.core.utils import checked_unite
 
 __all__ = ["Source", "Template", "InlineTemplate", "StaticFile", "DownloadSource"]
 
-import hashlib
 import os
+import time
 import urllib2
 import urlparse
 
@@ -136,36 +138,53 @@ else:
 
 
 class DownloadSource(Source):
-  def __init__(self, name, cache=True, md5sum=None):
+  """
+  redownload_files = True/False -- if file with the same name exists in tmp_dir
+  it won't be downloaded again (be if files are different this won't replace them)
+  
+  ignore_proxy = True/False -- determines if http_proxy / https_proxy environment variables
+  should be ignored or not
+  """
+    
+  def __init__(self, name, redownload_files=False, ignore_proxy=True):
     super(DownloadSource, self).__init__(name)
+
     self.url = self.name
-    self.md5sum = md5sum
-    self.cache = cache
-    if not 'download_path' in self.env.config:
-      self.env.config.download_path = '/var/tmp/downloads'
-    if not os.path.exists(self.env.config.download_path):
-      os.makedirs(self.env.config.download_path)
+    self.cache = not redownload_files and bool(self.env.tmp_dir)
+    self.download_path = self.env.tmp_dir
+    self.ignore_proxy = ignore_proxy
 
   def get_content(self):
-    filepath = os.path.basename(urlparse.urlparse(self.url).path)
-    content = None
-    if not self.cache or not os.path.exists(
-      os.path.join(self.env.config.download_path, filepath)):
-      web_file = urllib2.urlopen(self.url)
+    if self.download_path and not os.path.exists(self.download_path):
+      raise Fail("Directory {0} doesn't exist, please provide valid download path".format(self.download_path))
+    
+    if urlparse.urlparse(self.url).path:
+      filename = os.path.basename(urlparse.urlparse(self.url).path)
+    else:
+      filename = 'index.html.{0}'.format(time.time())
+    
+    if self.cache:  
+      filepath = os.path.join(self.download_path, filename)
+    
+    if not self.cache or not os.path.exists(filepath):
+      Logger.info("Downloading the file from {0}".format(self.url))
+      
+      if self.ignore_proxy:
+        opener = urllib2.build_opener(urllib2.ProxyHandler({}))
+      else:
+        opener = urllib2.build_opener()
+      
+      req = urllib2.Request(self.url)
+      web_file = opener.open(req)
       content = web_file.read()
+
+      if self.cache:
+        with open(filepath, 'w') as fp:
+          fp.write(content)
     else:
-      update = False
-      with open(os.path.join(self.env.config.download_path, filepath)) as fp:
+      Logger.info("Not downloading the file from {0}, because {1} already exists".format(self.url, filepath))
+        
+      with open(filepath) as fp:
         content = fp.read()
-      if self.md5sum:
-        m = hashlib.md5(content)
-        md5 = m.hexdigest()
-        if md5 != self.md5sum:
-          web_file = urllib2.urlopen(self.url)
-          content = web_file.read()
-          update = True
-      if self.cache and update:
-        with open(os.path.join(self.env.config.download_path, filepath),
-                  'w') as fp:
-          fp.write(content)
+
     return content

ambari-common/src/main/python/resource_management/libraries/functions/__init__.py

@@ -33,6 +33,7 @@ from resource_management.libraries.functions.get_port_from_url import *
 from resource_management.libraries.functions.hive_check import *
 from resource_management.libraries.functions.version import *
 from resource_management.libraries.functions.format_jvm_option import *
+from resource_management.libraries.functions.constants import *
 
 IS_WINDOWS = platform.system() == "Windows"
 

ambari-common/src/main/python/resource_management/core/constants.py → ambari-common/src/main/python/resource_management/libraries/functions/constants.py

@@ -18,6 +18,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
+__all__ = ["Direction", "SafeMode"]
+
 class Direction:
   """
   Rolling Upgrade direction

ambari-common/src/main/python/resource_management/libraries/script/__init__.py

@@ -22,4 +22,4 @@ Ambari Agent
 
 from resource_management.libraries.script.script import *
 from resource_management.libraries.script.hook import *
-from resource_management.libraries.script.config_dictionary import *
+from resource_management.libraries.script.config_dictionary import *

ambari-common/src/main/python/resource_management/libraries/script/script.py

@@ -180,7 +180,8 @@ class Script(object):
     # Run class method depending on a command type
     try:
       method = self.choose_method_to_execute(command_name)
-      with Environment(self.basedir) as env:
+      with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
+        env.config.download_path = Script.tmp_dir
         method(env)
 
         # For start actions, try to advertise the component's version

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py

@@ -31,15 +31,9 @@ def setup_ranger_hbase():
   import params
   
   if params.has_ranger_admin:
-
-    environment = {"no_proxy": format("{params.ambari_server_hostname}")}
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-            params.downloaded_custom_connector),
-            not_if=format("test -f {params.downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo=True)
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source)
+    )
 
     if not os.path.isfile(params.driver_curl_target):
       Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_upgrade.py

@@ -23,7 +23,7 @@ from resource_management.core.resources.system import Execute
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.default import default
 from resource_management.core.shell import call
-from resource_management.core.constants import Direction, SafeMode
+from resource_management.libraries.functions import Direction, SafeMode
 from resource_management.core.exceptions import Fail
 
 

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py

@@ -32,15 +32,9 @@ def setup_ranger_hdfs():
   import params
 
   if params.has_ranger_admin:
-
-    environment = {"no_proxy": format("{params.ambari_server_hostname}")}
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-            params.downloaded_custom_connector),
-            not_if=format("test -f {params.downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo=True)
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source)
+    )
 
     if not os.path.isfile(params.driver_curl_target):
       Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py

@@ -74,19 +74,10 @@ def hive(name=None):
 
   if name == 'metastore' or name == 'hiveserver2':
     jdbc_connector()
-    
-  environment = {
-    "no_proxy": format("{ambari_server_hostname}")
-  }
-
-  cmd = format("/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf -x \"\" "
-               "--retry 5 "
-               "{jdk_location}{check_db_connection_jar_name} "
-               "-o {check_db_connection_jar_name}'")
 
-  Execute(cmd,
-          not_if=format("[ -f {check_db_connection_jar} ]"),
-          environment = environment)
+  File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
+       content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
+  )
 
   if name == 'metastore':
     File(params.start_metastore_path,
@@ -205,14 +196,10 @@ def jdbc_connector():
     Execute(('rm', '-f', params.prepackaged_ojdbc_symlink),
             path=["/bin", "/usr/bin/"],
             sudo = True)
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-             params.downloaded_custom_connector),
-            not_if=format("test -f {downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo = True)
-
+    
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source),
+    )
 
     Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target),
             #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py

@@ -31,15 +31,9 @@ def setup_ranger_hive():
   import params
 
   if params.has_ranger_admin:
-
-    environment = {"no_proxy": format("{params.ambari_server_hostname}")}
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.ranger_driver_curl_source, '-o',
-            params.ranger_downloaded_custom_connector),
-            not_if=format("test -f {params.ranger_downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo=True)
+    File(params.ranger_downloaded_custom_connector,
+         content = DownloadSource(params.ranger_driver_curl_source),
+    )
 
     if not os.path.isfile(params.ranger_driver_curl_target):
       Execute(('cp', '--remove-destination', params.ranger_downloaded_custom_connector, params.ranger_driver_curl_target),

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py

@@ -31,15 +31,9 @@ def setup_ranger_knox():
   import params
 
   if params.has_ranger_admin:
-
-    environment = {"no_proxy": format("{params.ambari_server_hostname}")}
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-            params.downloaded_custom_connector),
-            not_if=format("test -f {params.downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo=True)
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source),
+    )
 
     if not os.path.isfile(params.driver_curl_target):
       Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py

@@ -76,20 +76,12 @@ def oozie(is_server=False # TODO: see if see can remove this
            group = params.user_group
     )
 
-  environment = {
-    "no_proxy": format("{ambari_server_hostname}")
-  }
-
   if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \
      params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
      params.jdbc_driver_name == "org.postgresql.Driver" or \
      params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
-    Execute(format("/bin/sh -c 'cd /usr/lib/ambari-agent/ &&\
-    curl -kf -x \"\" \
-    --retry 5 {jdk_location}{check_db_connection_jar_name}\
-     -o {check_db_connection_jar_name}'"),
-      not_if  = format("[ -f {check_db_connection_jar} ]"),
-      environment=environment
+    File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
+      content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
     )
   pass
 
@@ -157,17 +149,9 @@ def oozie_server_specific():
   if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \
      params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
      params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
-
-    environment = {
-      "no_proxy": format("{ambari_server_hostname}")
-    }
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-             params.downloaded_custom_connector),
-            not_if=format("test -f {downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo = True)
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source),
+    )
 
 
     Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target),

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py

@@ -24,9 +24,9 @@ import tempfile
 
 from resource_management.core import shell
 from resource_management.core.logger import Logger
-from resource_management.core.constants import Direction
 from resource_management.core.exceptions import Fail
 from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions import Direction
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions import compare_versions
 from resource_management.libraries.functions import format_hdp_stack_version

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py

@@ -28,15 +28,9 @@ def setup_ranger():
   import params
 
   if check_db_connnection():
-
-    environment = {"no_proxy": format("{params.ambari_server_hostname}")}
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-            params.downloaded_custom_connector),
-            not_if=format("test -f {params.downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo=True)
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source)
+    )
 
     if not os.path.isfile(params.driver_curl_target):
       Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py

@@ -32,15 +32,9 @@ def setup_ranger_storm():
   import params
 
   if params.has_ranger_admin and params.security_enabled:
-
-    environment = {"no_proxy": format("{params.ambari_server_hostname}")}
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-            params.downloaded_custom_connector),
-            not_if=format("test -f {params.downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo=True)
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source)
+    )
 
     if not os.path.isfile(params.driver_curl_target):
       Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py

@@ -29,20 +29,13 @@ def setup_jce():
   if not params.jdk_name:
     return
   
-  environment = {
-    "no_proxy": format("{ambari_server_hostname}")
-  }
-  
   if params.jce_policy_zip is not None:
     jce_curl_target = format("{artifact_dir}/{jce_policy_zip}")
-    download_jce = format("mkdir -p {artifact_dir}; \
-    curl -kf -x \"\" --retry 10 \
-    {jce_location}/{jce_policy_zip} -o {jce_curl_target}")
-    Execute( download_jce,
-             path = ["/bin","/usr/bin/"],
-             not_if =format("test -e {jce_curl_target}"),
-             ignore_failures = True,
-             environment = environment
+    Directory(params.artifact_dir,
+         recursive = True,
+    )
+    File(jce_curl_target,
+         content = DownloadSource(format("{jce_location}/{jce_policy_zip}")),
     )
   elif params.security_enabled:
     # Something weird is happening

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py

@@ -35,16 +35,12 @@ def setup_java():
   if not params.jdk_name:
     return
 
-  environment = {
-    "no_proxy": format("{ambari_server_hostname}")
-  }
-
-  Execute(format("mkdir -p {artifact_dir} ; \
-  curl -kf -x \"\" \
-  --retry 10 {jdk_location}/{jdk_name} -o {jdk_curl_target}"),
-          path = ["/bin","/usr/bin/"],
-          not_if = format("test -e {java_exec}"),
-          environment = environment)
+  Directory(params.artifact_dir,
+      recursive = True,
+  )
+  File(jdk_curl_target,
+       content = DownloadSource(format("{jdk_location}/{jdk_name}")),
+  )
 
   if params.jdk_name.endswith(".bin"):
     chmod_cmd = ("chmod", "+x", jdk_curl_target)

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py

@@ -97,23 +97,16 @@ def setup_database():
   Load DB
   """
   import params
-  db_driver_dload_cmd = ""
-  environment = {
-    "no_proxy": format("{ambari_server_hostname}")
-  }
+  db_driver_download_url = None
+  
   if params.server_db_name == 'oracle' and params.oracle_driver_url != "":
-    db_driver_dload_cmd = format(
-      "curl -kf -x \"\" \
-      --retry 5 {oracle_driver_symlink_url} -o {hadoop_lib_home}/{db_driver_filename}",)
+    db_driver_download_url = params.oracle_driver_symlink_url
   elif params.server_db_name == 'mysql' and params.mysql_driver_url != "":
-    db_driver_dload_cmd = format(
-      "curl -kf -x \"\" \
-      --retry 5 {mysql_driver_symlink_url} -o {hadoop_lib_home}/{db_driver_filename}")
-
-  if db_driver_dload_cmd:
-    Execute(db_driver_dload_cmd,
-            not_if =format("test -e {hadoop_lib_home}/{db_driver_filename}"),
-            environment = environment
+    db_driver_download_url = params.mysql_driver_symlink_url
+
+  if db_driver_download_url:
+    File(format("{hadoop_lib_home}/{db_driver_filename}"),
+         content = DownloadSource(db_driver_download_url),
     )
 
 

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py

@@ -81,9 +81,8 @@ class TestHiveClient(RMFTestCase):
                               owner = 'hive',
                               group = 'hadoop',
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-        environment = {'no_proxy': 'c6401.ambari.apache.org'},
-        not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
     )
     self.assertNoMoreResources()
 
@@ -146,8 +145,7 @@ class TestHiveClient(RMFTestCase):
                               owner = 'hive',
                               group = 'hadoop',
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-        environment = {'no_proxy': 'c6401.ambari.apache.org'},
-        not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
     )
     self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py

@@ -203,10 +203,9 @@ class TestHiveMetastore(RMFTestCase):
                               path = ['/bin', '/usr/bin/'],
                               sudo = True,
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-                              environment = {'no_proxy': u'c6401.ambari.apache.org'},
-                              not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
-                              )
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+    )
     self.assertResourceCalled('File', '/tmp/start_metastore_script',
                               content = StaticFile('startMetastore.sh'),
                               mode = 0755,
@@ -290,10 +289,9 @@ class TestHiveMetastore(RMFTestCase):
                               path = ['/bin', '/usr/bin/'],
                               sudo = True,
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-                              environment = {'no_proxy': u'c6401.ambari.apache.org'},
-                              not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
-                              )
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+    )
     self.assertResourceCalled('File', '/tmp/start_metastore_script',
                               content = StaticFile('startMetastore.sh'),
                               mode = 0755,

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py

@@ -323,10 +323,9 @@ class TestHiveServer(RMFTestCase):
                               path = ['/bin', '/usr/bin/'],
                               sudo = True,
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-                              environment = {'no_proxy': u'c6401.ambari.apache.org'},
-                              not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
-                              )
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+    )
     self.assertResourceCalled('File', '/tmp/start_hiveserver2_script',
                               content = Template('startHiveserver2.sh.j2'),
                               mode = 0755,
@@ -442,10 +441,9 @@ class TestHiveServer(RMFTestCase):
                               path = ['/bin', '/usr/bin/'],
                               sudo = True,
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-                              environment = {'no_proxy': u'c6401.ambari.apache.org'},
-                              not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
-                              )
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+    )
     self.assertResourceCalled('File', '/tmp/start_hiveserver2_script',
                               content = Template('startHiveserver2.sh.j2'),
                               mode = 0755,

ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py

@@ -30,11 +30,11 @@ class TestHookBeforeInstall(RMFTestCase):
                        command="hook",
                        config_file="default.json"
     )
-    self.assertResourceCalled('Execute', 'mkdir -p /tmp/AMBARI-artifacts/;     curl -kf -x "" --retry 10     http://c6401.ambari.apache.org:8080/resources//UnlimitedJCEPolicyJDK7.zip -o /tmp/AMBARI-artifacts//UnlimitedJCEPolicyJDK7.zip',
-        environment = {'no_proxy': 'c6401.ambari.apache.org'},
-        not_if = 'test -e /tmp/AMBARI-artifacts//UnlimitedJCEPolicyJDK7.zip',
-        ignore_failures = True,
-        path = ['/bin', '/usr/bin/'],
+    self.assertResourceCalled('Directory', '/tmp/AMBARI-artifacts/',
+        recursive = True,
+    )
+    self.assertResourceCalled('File', '/tmp/AMBARI-artifacts//UnlimitedJCEPolicyJDK7.zip',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//UnlimitedJCEPolicyJDK7.zip'),
     )
     self.assertResourceCalled('Group', 'hadoop',
         ignore_failures = False,

ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py

@@ -18,9 +18,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
-from stacks.utils.RMFTestCase import *
 from mock.mock import MagicMock, call, patch
 from resource_management import *
+from stacks.utils.RMFTestCase import *
 
 @patch.object(Hook, "run_custom_hook", new = MagicMock())
 class TestHookBeforeInstall(RMFTestCase):
@@ -40,10 +40,11 @@ class TestHookBeforeInstall(RMFTestCase):
     )
     self.assertResourceCalled('Package', 'unzip',)
     self.assertResourceCalled('Package', 'curl',)
-    self.assertResourceCalled('Execute', 'mkdir -p /tmp/AMBARI-artifacts/ ;   curl -kf -x \"\"   --retry 10 http://c6401.ambari.apache.org:8080/resources//jdk-7u67-linux-x64.tar.gz -o /tmp/AMBARI-artifacts//jdk-7u67-linux-x64.tar.gz',
-        not_if = 'test -e /usr/jdk64/jdk1.7.0_45/bin/java',
-        path = ['/bin', '/usr/bin/'],
-        environment = {'no_proxy': 'c6401.ambari.apache.org'},
+    self.assertResourceCalled('Directory', '/tmp/AMBARI-artifacts/',
+        recursive = True,
+    )
+    self.assertResourceCalled('File', '/tmp/AMBARI-artifacts//jdk-7u67-linux-x64.tar.gz',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//jdk-7u67-linux-x64.tar.gz'),
     )
     self.assertResourceCalled('Directory', '/usr/jdk64',)
     self.assertResourceCalled('Execute', ('chmod', 'a+x', u'/usr/jdk64'),

ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py

@@ -193,10 +193,9 @@ class TestHiveMetastore(RMFTestCase):
                               path = ['/bin', '/usr/bin/'],
                               sudo = True,
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-                              environment = {'no_proxy': u'c6401.ambari.apache.org'},
-                              not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
-                              )
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+    )
     self.assertResourceCalled('File', '/tmp/start_metastore_script',
                               content = StaticFile('startMetastore.sh'),
                               mode = 0755,
@@ -271,10 +270,9 @@ class TestHiveMetastore(RMFTestCase):
                               path = ['/bin', '/usr/bin/'],
                               sudo = True,
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-                              environment = {'no_proxy': u'c6401.ambari.apache.org'},
-                              not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
-                              )
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+    )
     self.assertResourceCalled('File', '/tmp/start_metastore_script',
                               content = StaticFile('startMetastore.sh'),
                               mode = 0755,

ambari-server/src/test/python/stacks/utils/RMFTestCase.py

@@ -17,7 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
-__all__ = ["RMFTestCase", "Template", "StaticFile", "InlineTemplate", "UnknownConfigurationMock", "FunctionMock",
+__all__ = ["RMFTestCase", "Template", "StaticFile", "InlineTemplate", "DownloadSource", "UnknownConfigurationMock", "FunctionMock",
            "CallFunctionMock"]
 
 from unittest import TestCase
@@ -244,7 +244,14 @@ def InlineTemplate(name, **kwargs):
   with RMFTestCase.env:
     from resource_management.core.source import InlineTemplate
     return InlineTemplate(name, **kwargs)
-
+  
+class DownloadSource():
+  def __init__(self, name, **kwargs):
+    self.name = name
+  
+  def __eq__(self, other):
+    from resource_management.core.source import DownloadSource
+    return isinstance(other, DownloadSource) and self.name == other.name
 
 class UnknownConfigurationMock():
   def __eq__(self, other):