AMBARI-9581. curl unable to connect to Ambari when SSLv3 and TLSv1 is disabled (aonishuk)

ambari-agent/src/main/python/ambari_agent/AlertSchedulerHandler.py

@@ -53,7 +53,7 @@ class AlertSchedulerHandler():
   }
 
 
-  def __init__(self, cachedir, stacks_dir, common_services_dir, host_scripts_dir, in_minutes=True):
+  def __init__(self, cachedir, stacks_dir, common_services_dir, host_scripts_dir, config, in_minutes=True):
     self.cachedir = cachedir
     self.stacks_dir = stacks_dir
     self.common_services_dir = common_services_dir
@@ -69,6 +69,7 @@ class AlertSchedulerHandler():
     self._collector = AlertCollector()
     self.__scheduler = Scheduler(AlertSchedulerHandler.APS_CONFIG)
     self.__in_minutes = in_minutes
+    self.config = config
     self.__config_maps = {}
 
 
@@ -281,7 +282,7 @@ class AlertSchedulerHandler():
       source['stacks_directory'] = self.stacks_dir
       source['common_services_directory'] = self.common_services_dir
       source['host_scripts_directory'] = self.host_scripts_dir
-      alert = ScriptAlert(json_definition, source)
+      alert = ScriptAlert(json_definition, source, self.config)
     elif source_type == AlertSchedulerHandler.TYPE_WEB:
       alert = WebAlert(json_definition, source)
 

ambari-agent/src/main/python/ambari_agent/Controller.py

@@ -91,7 +91,7 @@ class Controller(threading.Thread):
     alerts_cache_dir = os.path.join(cache_dir, 'alerts')
     
     self.alert_scheduler_handler = AlertSchedulerHandler(alerts_cache_dir, 
-        stacks_cache_dir, common_services_cache_dir, host_scripts_cache_dir)
+        stacks_cache_dir, common_services_cache_dir, host_scripts_cache_dir, config)
 
 
   def __del__(self):

ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py

@@ -29,7 +29,7 @@ from symbol import parameters
 logger = logging.getLogger()
 
 class ScriptAlert(BaseAlert):
-  def __init__(self, alert_meta, alert_source_meta):
+  def __init__(self, alert_meta, alert_source_meta, config):
     """ ScriptAlert reporting structure is output from the script itself """
     
     alert_source_meta['reporting'] = {
@@ -41,6 +41,7 @@ class ScriptAlert(BaseAlert):
     
     super(ScriptAlert, self).__init__(alert_meta, alert_source_meta)
     
+    self.config = config
     self.path = None
     self.stacks_dir = None
     self.common_services_dir = None
@@ -90,7 +91,7 @@ class ScriptAlert(BaseAlert):
       matchObj = re.match( r'((.*)services\/(.*)\/package\/)', self.path_to_script)
       if matchObj:
         basedir = matchObj.group(1)
-        with Environment(basedir) as env:
+        with Environment(basedir, tmp_dir=self.config.get.get('agent', 'tmp_dir')) as env:
           return cmd_module.execute(parameters, self.host_name)
       else:
         return cmd_module.execute(parameters, self.host_name)

ambari-agent/src/test/python/ambari_agent/TestAlerts.py

@@ -55,7 +55,7 @@ class TestAlerts(TestCase):
     test_common_services_path = os.path.join('ambari_agent', 'dummy_files')
     test_host_scripts_path = os.path.join('ambari_agent', 'dummy_files')
 
-    ash = AlertSchedulerHandler(test_file_path, test_stack_path, test_common_services_path, test_host_scripts_path)
+    ash = AlertSchedulerHandler(test_file_path, test_stack_path, test_common_services_path, test_host_scripts_path, None)
     ash.start()
 
     self.assertTrue(aps_add_interval_job_mock.called)
@@ -221,7 +221,7 @@ class TestAlerts(TestCase):
     json['source']['host_scripts_directory'] = os.path.join('ambari_agent', 'host_scripts')
 
     collector = AlertCollector()
-    sa = ScriptAlert(json, json['source'])
+    sa = ScriptAlert(json, json['source'], MagicMock())
     sa.set_helpers(collector, {'foo-site/bar': 'rendered-bar', 'foo-site/baz':'rendered-baz'} )
     self.assertEquals(json['source']['path'], sa.path)
     self.assertEquals(json['source']['stacks_directory'], sa.stacks_dir)
@@ -490,7 +490,7 @@ class TestAlerts(TestCase):
     test_common_services_path = os.path.join('ambari_agent', 'dummy_files')
     test_host_scripts_path = os.path.join('ambari_agent', 'dummy_files')
     
-    ash = AlertSchedulerHandler(test_file_path, test_stack_path, test_common_services_path, test_host_scripts_path)
+    ash = AlertSchedulerHandler(test_file_path, test_stack_path, test_common_services_path, test_host_scripts_path, None)
     ash.start()
 
     self.assertEquals(1, ash.get_job_count())
@@ -546,7 +546,7 @@ class TestAlerts(TestCase):
     test_common_services_path = os.path.join('ambari_agent', 'dummy_files')
     test_host_scripts_path = os.path.join('ambari_agent', 'dummy_files')
 
-    ash = AlertSchedulerHandler(test_file_path, test_stack_path, test_common_services_path, test_host_scripts_path)
+    ash = AlertSchedulerHandler(test_file_path, test_stack_path, test_common_services_path, test_host_scripts_path, None)
     ash.start()
 
     self.assertEquals(1, ash.get_job_count())
@@ -599,7 +599,7 @@ class TestAlerts(TestCase):
     test_common_services_path = os.path.join('ambari_agent', 'dummy_files')
     test_host_scripts_path = os.path.join('ambari_agent', 'dummy_files')
 
-    ash = AlertSchedulerHandler(test_file_path, test_stack_path, test_common_services_path, test_host_scripts_path)
+    ash = AlertSchedulerHandler(test_file_path, test_stack_path, test_common_services_path, test_host_scripts_path, None)
     ash.start()
 
     self.assertEquals(1, ash.get_job_count())
@@ -660,7 +660,7 @@ class TestAlerts(TestCase):
     json['source']['host_scripts_directory'] = os.path.join('ambari_agent', 'host_scripts')
 
     collector = AlertCollector()
-    sa = ScriptAlert(json, json['source'])
+    sa = ScriptAlert(json, json['source'], None)
 
     # instruct the test alert script to be skipped
     sa.set_helpers(collector, {'foo-site/skip': 'true'} )
@@ -690,7 +690,7 @@ class TestAlerts(TestCase):
       }
     }
 
-    alert = ScriptAlert(json, json['source'])
+    alert = ScriptAlert(json, json['source'], None)
     self.assertEquals(alert._get_reporting_text(alert.RESULT_OK), '{0}')
     self.assertEquals(alert._get_reporting_text(alert.RESULT_WARNING), '{0}')
     self.assertEquals(alert._get_reporting_text(alert.RESULT_CRITICAL), '{0}')
@@ -738,7 +738,7 @@ class TestAlerts(TestCase):
       all_commands = json.load(fp)
     all_commands[0]['configurations']['hdfs-site'].update({"dfs.namenode.http-address": "c6401.ambari.apache.org:50071"})
 
-    ash = AlertSchedulerHandler(test_file_path, test_stack_path, test_common_services_path, test_host_scripts_path)
+    ash = AlertSchedulerHandler(test_file_path, test_stack_path, test_common_services_path, test_host_scripts_path, None)
     ash.start()
 
     with patch("__builtin__.open") as open_mock:

ambari-agent/src/test/python/resource_management/TestContentSources.py

@@ -75,114 +75,62 @@ class TestContentSources(TestCase):
     join_mock.assert_called_with('/base', 'files', 'relative/path/file')
     self.assertEqual(open_mock.call_count, 1)
 
-  @patch.object(os, "makedirs")
-  @patch.object(os.path, "exists")
-  def test_download_source_init_existent_download_directory(self, exists_mock, makedirs_mock):
-    """
-    Testing DownloadSource without cache with existent download directory
-    """
-    exists_mock.return_value = True
-
-    with Environment("/base") as env:
-      static_file = DownloadSource("http://download/source")
-
-    self.assertEqual(makedirs_mock.call_count, 0)
-    self.assertEqual(exists_mock.call_count, 1)
-    pass
-
-
-  @patch.object(os, "makedirs")
-  @patch.object(os.path, "exists")
-  def test_download_source_init_nonexistent_download_directory(self, exists_mock, makedirs_mock):
-    """
-    Testing DownloadSource without cache with non-existent download directory
-    """
-    exists_mock.return_value = False
-
-    with Environment("/base") as env:
-      static_file = DownloadSource("http://download/source")
-
-    self.assertEqual(makedirs_mock.call_count, 1)
-    makedirs_mock.assert_called_with("/var/tmp/downloads")
-    self.assertEqual(exists_mock.call_count, 1)
-    pass
-
-  @patch.object(urllib2, "urlopen")
-  @patch.object(os, "makedirs")
+  @patch.object(urllib2, "build_opener")
+  @patch.object(urllib2, "Request")
   @patch.object(os.path, "exists")
-  def test_download_source_get_content_nocache(self, exists_mock, makedirs_mock, urlopen_mock):
+  def test_download_source_get_content_nocache(self, exists_mock, request_mock, opener_mock):
     """
     Testing DownloadSource.get_content without cache
     """
     exists_mock.return_value = True
     web_file_mock = MagicMock()
     web_file_mock.read.return_value = 'web_content'
-    urlopen_mock.return_value = web_file_mock
+    opener_mock.return_value.open = MagicMock(return_value=web_file_mock)
 
-    with Environment("/base") as env:
-      download_source = DownloadSource("http://download/source", cache=False)
+    with Environment("/base", tmp_dir='/var/tmp/downloads') as env:
+      download_source = DownloadSource("http://download/source", redownload_files=True)
     content = download_source.get_content()
 
     self.assertEqual('web_content', content)
-    self.assertEqual(urlopen_mock.call_count, 1)
-    urlopen_mock.assert_called_with('http://download/source')
+    self.assertEqual(opener_mock.call_count, 1)
+    request_mock.assert_called_with('http://download/source')
     self.assertEqual(web_file_mock.read.call_count, 1)
-
-  @patch.object(urllib2, "urlopen")
+    
+  @patch("__builtin__.open")
+  @patch.object(urllib2, "Request")
+  @patch.object(urllib2, "build_opener")
   @patch.object(os, "makedirs")
   @patch.object(os.path, "exists")
-  def test_download_source_get_content_cache_new(self, exists_mock, makedirs_mock, urlopen_mock):
+  def test_download_source_get_content_cache_new(self, exists_mock, makedirs_mock, opener_mock, request_mock, open_mock):
     """
     Testing DownloadSource.get_content with cache on non-cached resource
     """
     exists_mock.side_effect = [True, False]
     web_file_mock = MagicMock()
     web_file_mock.read.return_value = 'web_content'
-    urlopen_mock.return_value = web_file_mock
-
-    with Environment("/base") as env:
-      download_source = DownloadSource("http://download/source", cache=True)
-    content = download_source.get_content()
-
-    self.assertEqual('web_content', content)
-    self.assertEqual(urlopen_mock.call_count, 1)
-    urlopen_mock.assert_called_with('http://download/source')
-    self.assertEqual(web_file_mock.read.call_count, 1)
-
-  @patch("__builtin__.open")
-  @patch.object(os, "makedirs")
-  @patch.object(os.path, "exists")
-  def test_download_source_get_content_cache_existent(self, exists_mock, makedirs_mock, open_mock):
-    """
-    Testing DownloadSource.get_content with cache on cached resource
-    """
-    exists_mock.side_effect = [True, True, False]
-
+    opener_mock.return_value.open = MagicMock(return_value=web_file_mock)
+    
     file_mock = MagicMock(name = 'file_mock')
     file_mock.__enter__.return_value = file_mock
-    file_mock.read.return_value = 'cached_content'
+    file_mock.read.return_value = 'content'
     open_mock.return_value = file_mock
 
-
-    with Environment("/base") as env:
-      download_source = DownloadSource("http://download/source", cache=True)
+    with Environment("/base", tmp_dir='/var/tmp/downloads') as env:
+      download_source = DownloadSource("http://download/source", redownload_files=False)
     content = download_source.get_content()
 
-    self.assertEqual('cached_content', content)
-    self.assertEqual(open_mock.call_count, 1)
-    open_mock.assert_called_with('/var/tmp/downloads/source')
-    self.assertEqual(file_mock.read.call_count, 1)
+    self.assertEqual('web_content', content)
+    self.assertEqual(opener_mock.call_count, 1)
+    request_mock.assert_called_with('http://download/source')
+    self.assertEqual(web_file_mock.read.call_count, 1)
 
-  @patch.object(urllib2, "urlopen")
   @patch("__builtin__.open")
-  @patch.object(os, "makedirs")
   @patch.object(os.path, "exists")
-  def test_download_source_get_content_cache_existent_md5_match(self, exists_mock, makedirs_mock, open_mock,
-                                                                urlopen_mock):
+  def test_download_source_get_content_cache_existent(self, exists_mock, open_mock):
     """
-    Testing DownloadSource.get_content with cache on cached resource with md5 check
+    Testing DownloadSource.get_content with cache on cached resource
     """
-    exists_mock.side_effect = [True, True, False]
+    exists_mock.side_effect = [True, True]
 
     file_mock = MagicMock(name = 'file_mock')
     file_mock.__enter__.return_value = file_mock
@@ -190,47 +138,14 @@ class TestContentSources(TestCase):
     open_mock.return_value = file_mock
 
 
-    with Environment("/base") as env:
-      download_source = DownloadSource("http://download/source", cache=True)
+    with Environment("/base", tmp_dir='/var/tmp/downloads') as env:
+      download_source = DownloadSource("http://download/source", redownload_files=False)
     content = download_source.get_content()
 
     self.assertEqual('cached_content', content)
     self.assertEqual(open_mock.call_count, 1)
-    open_mock.assert_called_with('/var/tmp/downloads/source')
-    self.assertEqual(file_mock.read.call_count, 1)
-    self.assertEqual(urlopen_mock.call_count, 0)
-
-  @patch.object(urllib2, "urlopen")
-  @patch("__builtin__.open")
-  @patch.object(os, "makedirs")
-  @patch.object(os.path, "exists")
-  def test_download_source_get_content_cache_existent_md5_unmatch(self, exists_mock, makedirs_mock, open_mock,
-                                                                  urlopen_mock):
-    """
-    Testing DownloadSource.get_content with cache on cached resource with md5 check
-    """
-    exists_mock.side_effect = [True, True, False]
-    fake_md5 = "144c9defac04969c7bfad8efaa8ea194"
-    file_mock = MagicMock(name = 'file_mock')
-    file_mock.__enter__.return_value = file_mock
-    file_mock.read.return_value = 'cached_content'
-    open_mock.return_value = file_mock
-    web_file_mock = MagicMock()
-    web_file_mock.read.return_value = 'web_content'
-    urlopen_mock.return_value = web_file_mock
-
-    with Environment("/base") as env:
-      download_source = DownloadSource("http://download/source", cache=True, md5sum=fake_md5)
-    content = download_source.get_content()
-
-    self.assertEqual('web_content', content)
-    self.assertEqual(open_mock.call_count, 2)
-    open_mock.assert_once_called('/var/tmp/downloads/source', 'w')
-    open_mock.assert_once_called('/var/tmp/downloads/source')
     self.assertEqual(file_mock.read.call_count, 1)
-    self.assertEqual(urlopen_mock.call_count, 1)
-    urlopen_mock.assert_called_with('http://download/source')
-
+    
   @patch("__builtin__.open")
   @patch.object(os.path, "getmtime")
   @patch.object(os.path, "exists")

ambari-common/src/main/python/resource_management/core/environment.py

@@ -39,24 +39,25 @@ from resource_management.core.logger import Logger
 class Environment(object):
   _instances = []
 
-  def __init__(self, basedir=None, test_mode=False, logging_level=logging.INFO):
+  def __init__(self, basedir=None, tmp_dir=None, test_mode=False, logging_level=logging.INFO):
     """
     @param basedir: basedir/files, basedir/templates are the places where templates / static files
     are looked up
     @param test_mode: if this is enabled, resources won't be executed until manualy running env.run().
-    """
-    self.reset(basedir, test_mode)
+    """   
+    self.reset(basedir, test_mode, tmp_dir)
     
     if not Logger.logger:
       Logger.initialize_logger(logging_level)
 
-  def reset(self, basedir, test_mode):
+  def reset(self, basedir, test_mode, tmp_dir):
     self.system = System.get_instance()
     self.config = AttributeDictionary()
     self.resources = {}
     self.resource_list = []
     self.delayed_actions = set()
     self.test_mode = test_mode
+    self.tmp_dir = tmp_dir
     self.update_config({
       # current time
       'date': datetime.now(),

ambari-common/src/main/python/resource_management/core/source.py

@@ -22,12 +22,14 @@ Ambari Agent
 
 from __future__ import with_statement
 from resource_management.core.environment import Environment
+from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
 from resource_management.core.utils import checked_unite
 
 __all__ = ["Source", "Template", "InlineTemplate", "StaticFile", "DownloadSource"]
 
-import hashlib
 import os
+import time
 import urllib2
 import urlparse
 
@@ -136,36 +138,52 @@ else:
 
 
 class DownloadSource(Source):
-  def __init__(self, name, cache=True, md5sum=None):
+  """
+  redownload_files = True/False -- if file with the same name exists in tmp_dir
+  it won't be downloaded again (be if files are different this won't replace them)
+  
+  ignore_proxy = True/False -- determines if http_proxy / https_proxy environment variables
+  should be ignored or not
+  """
+    
+  def __init__(self, name, redownload_files=False, ignore_proxy=True):
     super(DownloadSource, self).__init__(name)
+
     self.url = self.name
-    self.md5sum = md5sum
-    self.cache = cache
-    if not 'download_path' in self.env.config:
-      self.env.config.download_path = '/var/tmp/downloads'
-    if not os.path.exists(self.env.config.download_path):
-      os.makedirs(self.env.config.download_path)
+    self.cache = not redownload_files and bool(self.env.tmp_dir)
+    self.download_path = self.env.tmp_dir
+    self.ignore_proxy = ignore_proxy
 
   def get_content(self):
-    filepath = os.path.basename(urlparse.urlparse(self.url).path)
-    content = None
-    if not self.cache or not os.path.exists(
-      os.path.join(self.env.config.download_path, filepath)):
-      web_file = urllib2.urlopen(self.url)
+    if self.download_path and not os.path.exists(self.download_path):
+      raise Fail("Directory {0} doesn't exist, please provide valid download path".format(self.download_path))
+    
+    if urlparse.urlparse(self.url).path:
+      filename = os.path.basename(urlparse.urlparse(self.url).path)
+    else:
+      filename = 'index.html.{0}'.format(time.time())
+      
+    filepath = os.path.join(self.download_path, filename)
+    
+    if not self.cache or not os.path.exists(filepath):
+      Logger.info("Downloading the file from {0}".format(self.url))
+      
+      if self.ignore_proxy:
+        opener = urllib2.build_opener(urllib2.ProxyHandler({}))
+      else:
+        opener = urllib2.build_opener()
+      
+      req = urllib2.Request(self.url)
+      web_file = opener.open(req)
       content = web_file.read()
+
+      if self.cache:
+        with open(filepath, 'w') as fp:
+          fp.write(content)
     else:
-      update = False
-      with open(os.path.join(self.env.config.download_path, filepath)) as fp:
+      Logger.info("Not downloading the file from {0}, because {1} already exists".format(self.url, filepath))
+        
+      with open(filepath) as fp:
         content = fp.read()
-      if self.md5sum:
-        m = hashlib.md5(content)
-        md5 = m.hexdigest()
-        if md5 != self.md5sum:
-          web_file = urllib2.urlopen(self.url)
-          content = web_file.read()
-          update = True
-      if self.cache and update:
-        with open(os.path.join(self.env.config.download_path, filepath),
-                  'w') as fp:
-          fp.write(content)
+
     return content

ambari-common/src/main/python/resource_management/libraries/functions/__init__.py

@@ -33,6 +33,7 @@ from resource_management.libraries.functions.get_port_from_url import *
 from resource_management.libraries.functions.hive_check import *
 from resource_management.libraries.functions.version import *
 from resource_management.libraries.functions.format_jvm_option import *
+from resource_management.libraries.functions.constants import *
 
 IS_WINDOWS = platform.system() == "Windows"
 

ambari-common/src/main/python/resource_management/core/constants.py → ambari-common/src/main/python/resource_management/libraries/functions/constants.py

@@ -18,6 +18,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
+__all__ = ["Direction", "SafeMode"]
+
 class Direction:
   """
   Rolling Upgrade direction

ambari-common/src/main/python/resource_management/libraries/script/__init__.py

@@ -22,4 +22,4 @@ Ambari Agent
 
 from resource_management.libraries.script.script import *
 from resource_management.libraries.script.hook import *
-from resource_management.libraries.script.config_dictionary import *
+from resource_management.libraries.script.config_dictionary import *

ambari-common/src/main/python/resource_management/libraries/script/script.py

@@ -180,7 +180,8 @@ class Script(object):
     # Run class method depending on a command type
     try:
       method = self.choose_method_to_execute(command_name)
-      with Environment(self.basedir) as env:
+      with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
+        env.config.download_path = Script.tmp_dir
         method(env)
 
         # For start actions, try to advertise the component's version

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py

@@ -31,15 +31,9 @@ def setup_ranger_hbase():
   import params
   
   if params.has_ranger_admin:
-
-    environment = {"no_proxy": format("{params.ambari_server_hostname}")}
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-            params.downloaded_custom_connector),
-            not_if=format("test -f {params.downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo=True)
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source)
+    )
 
     if not os.path.isfile(params.driver_curl_target):
       Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_upgrade.py

@@ -23,7 +23,7 @@ from resource_management.core.resources.system import Execute
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.default import default
 from resource_management.core.shell import call
-from resource_management.core.constants import Direction, SafeMode
+from resource_management.libraries.functions import Direction, SafeMode
 from resource_management.core.exceptions import Fail
 
 

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py

@@ -32,15 +32,9 @@ def setup_ranger_hdfs():
   import params
 
   if params.has_ranger_admin:
-
-    environment = {"no_proxy": format("{params.ambari_server_hostname}")}
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-            params.downloaded_custom_connector),
-            not_if=format("test -f {params.downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo=True)
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source)
+    )
 
     if not os.path.isfile(params.driver_curl_target):
       Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py

@@ -74,19 +74,10 @@ def hive(name=None):
 
   if name == 'metastore' or name == 'hiveserver2':
     jdbc_connector()
-    
-  environment = {
-    "no_proxy": format("{ambari_server_hostname}")
-  }
-
-  cmd = format("/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf -x \"\" "
-               "--retry 5 "
-               "{jdk_location}{check_db_connection_jar_name} "
-               "-o {check_db_connection_jar_name}'")
 
-  Execute(cmd,
-          not_if=format("[ -f {check_db_connection_jar} ]"),
-          environment = environment)
+  File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
+       content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
+  )
 
   if name == 'metastore':
     File(params.start_metastore_path,
@@ -205,14 +196,10 @@ def jdbc_connector():
     Execute(('rm', '-f', params.prepackaged_ojdbc_symlink),
             path=["/bin", "/usr/bin/"],
             sudo = True)
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-             params.downloaded_custom_connector),
-            not_if=format("test -f {downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo = True)
-
+    
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source),
+    )
 
     Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target),
             #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py

@@ -31,15 +31,9 @@ def setup_ranger_hive():
   import params
 
   if params.has_ranger_admin:
-
-    environment = {"no_proxy": format("{params.ambari_server_hostname}")}
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.ranger_driver_curl_source, '-o',
-            params.ranger_downloaded_custom_connector),
-            not_if=format("test -f {params.ranger_downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo=True)
+    File(params.ranger_downloaded_custom_connector,
+         content = DownloadSource(params.ranger_driver_curl_source),
+    )
 
     if not os.path.isfile(params.ranger_driver_curl_target):
       Execute(('cp', '--remove-destination', params.ranger_downloaded_custom_connector, params.ranger_driver_curl_target),

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py

@@ -31,15 +31,9 @@ def setup_ranger_knox():
   import params
 
   if params.has_ranger_admin:
-
-    environment = {"no_proxy": format("{params.ambari_server_hostname}")}
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-            params.downloaded_custom_connector),
-            not_if=format("test -f {params.downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo=True)
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source),
+    )
 
     if not os.path.isfile(params.driver_curl_target):
       Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py

@@ -76,20 +76,12 @@ def oozie(is_server=False # TODO: see if see can remove this
            group = params.user_group
     )
 
-  environment = {
-    "no_proxy": format("{ambari_server_hostname}")
-  }
-
   if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \
      params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
      params.jdbc_driver_name == "org.postgresql.Driver" or \
      params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
-    Execute(format("/bin/sh -c 'cd /usr/lib/ambari-agent/ &&\
-    curl -kf -x \"\" \
-    --retry 5 {jdk_location}{check_db_connection_jar_name}\
-     -o {check_db_connection_jar_name}'"),
-      not_if  = format("[ -f {check_db_connection_jar} ]"),
-      environment=environment
+    File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
+      content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
     )
   pass
 
@@ -157,17 +149,9 @@ def oozie_server_specific():
   if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \
      params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
      params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
-
-    environment = {
-      "no_proxy": format("{ambari_server_hostname}")
-    }
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-             params.downloaded_custom_connector),
-            not_if=format("test -f {downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo = True)
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source),
+    )
 
 
     Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target),

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py

@@ -24,9 +24,9 @@ import tempfile
 
 from resource_management.core import shell
 from resource_management.core.logger import Logger
-from resource_management.core.constants import Direction
 from resource_management.core.exceptions import Fail
 from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions import Direction
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions import compare_versions
 from resource_management.libraries.functions import format_hdp_stack_version

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger.py

@@ -28,15 +28,9 @@ def setup_ranger():
   import params
 
   if check_db_connnection():
-
-    environment = {"no_proxy": format("{params.ambari_server_hostname}")}
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-            params.downloaded_custom_connector),
-            not_if=format("test -f {params.downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo=True)
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source)
+    )
 
     if not os.path.isfile(params.driver_curl_target):
       Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_ranger_storm.py

@@ -32,15 +32,9 @@ def setup_ranger_storm():
   import params
 
   if params.has_ranger_admin and params.security_enabled:
-
-    environment = {"no_proxy": format("{params.ambari_server_hostname}")}
-
-    Execute(('curl', '-kf', '-x', "", '--retry', '10', params.driver_curl_source, '-o',
-            params.downloaded_custom_connector),
-            not_if=format("test -f {params.downloaded_custom_connector}"),
-            path=["/bin", "/usr/bin/"],
-            environment=environment,
-            sudo=True)
+    File(params.downloaded_custom_connector,
+         content = DownloadSource(params.driver_curl_source)
+    )
 
     if not os.path.isfile(params.driver_curl_target):
       Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.driver_curl_target),

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py

@@ -29,20 +29,13 @@ def setup_jce():
   if not params.jdk_name:
     return
   
-  environment = {
-    "no_proxy": format("{ambari_server_hostname}")
-  }
-  
   if params.jce_policy_zip is not None:
     jce_curl_target = format("{artifact_dir}/{jce_policy_zip}")
-    download_jce = format("mkdir -p {artifact_dir}; \
-    curl -kf -x \"\" --retry 10 \
-    {jce_location}/{jce_policy_zip} -o {jce_curl_target}")
-    Execute( download_jce,
-             path = ["/bin","/usr/bin/"],
-             not_if =format("test -e {jce_curl_target}"),
-             ignore_failures = True,
-             environment = environment
+    Directory(params.artifact_dir,
+         recursive = True,
+    )
+    File(jce_curl_target,
+         content = DownloadSource(format("{jce_location}/{jce_policy_zip}")),
     )
   elif params.security_enabled:
     # Something weird is happening

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py

@@ -35,16 +35,12 @@ def setup_java():
   if not params.jdk_name:
     return
 
-  environment = {
-    "no_proxy": format("{ambari_server_hostname}")
-  }
-
-  Execute(format("mkdir -p {artifact_dir} ; \
-  curl -kf -x \"\" \
-  --retry 10 {jdk_location}/{jdk_name} -o {jdk_curl_target}"),
-          path = ["/bin","/usr/bin/"],
-          not_if = format("test -e {java_exec}"),
-          environment = environment)
+  Directory(params.artifact_dir,
+      recursive = True,
+  )
+  File(jdk_curl_target,
+       content = DownloadSource(format("{jdk_location}/{jdk_name}")),
+  )
 
   if params.jdk_name.endswith(".bin"):
     chmod_cmd = ("chmod", "+x", jdk_curl_target)

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py

@@ -97,23 +97,16 @@ def setup_database():
   Load DB
   """
   import params
-  db_driver_dload_cmd = ""
-  environment = {
-    "no_proxy": format("{ambari_server_hostname}")
-  }
+  db_driver_download_url = None
+  
   if params.server_db_name == 'oracle' and params.oracle_driver_url != "":
-    db_driver_dload_cmd = format(
-      "curl -kf -x \"\" \
-      --retry 5 {oracle_driver_symlink_url} -o {hadoop_lib_home}/{db_driver_filename}",)
+    db_driver_download_url = params.oracle_driver_symlink_url
   elif params.server_db_name == 'mysql' and params.mysql_driver_url != "":
-    db_driver_dload_cmd = format(
-      "curl -kf -x \"\" \
-      --retry 5 {mysql_driver_symlink_url} -o {hadoop_lib_home}/{db_driver_filename}")
-
-  if db_driver_dload_cmd:
-    Execute(db_driver_dload_cmd,
-            not_if =format("test -e {hadoop_lib_home}/{db_driver_filename}"),
-            environment = environment
+    db_driver_download_url = params.mysql_driver_symlink_url
+
+  if db_driver_download_url:
+    File(format("{hadoop_lib_home}/{db_driver_filename}"),
+         content = DownloadSource(db_driver_download_url),
     )
 
 

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py

@@ -81,9 +81,8 @@ class TestHiveClient(RMFTestCase):
                               owner = 'hive',
                               group = 'hadoop',
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-        environment = {'no_proxy': 'c6401.ambari.apache.org'},
-        not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
     )
     self.assertNoMoreResources()
 
@@ -146,8 +145,7 @@ class TestHiveClient(RMFTestCase):
                               owner = 'hive',
                               group = 'hadoop',
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-        environment = {'no_proxy': 'c6401.ambari.apache.org'},
-        not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
     )
     self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py

@@ -203,10 +203,9 @@ class TestHiveMetastore(RMFTestCase):
                               path = ['/bin', '/usr/bin/'],
                               sudo = True,
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-                              environment = {'no_proxy': u'c6401.ambari.apache.org'},
-                              not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
-                              )
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+    )
     self.assertResourceCalled('File', '/tmp/start_metastore_script',
                               content = StaticFile('startMetastore.sh'),
                               mode = 0755,
@@ -290,10 +289,9 @@ class TestHiveMetastore(RMFTestCase):
                               path = ['/bin', '/usr/bin/'],
                               sudo = True,
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-                              environment = {'no_proxy': u'c6401.ambari.apache.org'},
-                              not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
-                              )
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+    )
     self.assertResourceCalled('File', '/tmp/start_metastore_script',
                               content = StaticFile('startMetastore.sh'),
                               mode = 0755,

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py

@@ -323,10 +323,9 @@ class TestHiveServer(RMFTestCase):
                               path = ['/bin', '/usr/bin/'],
                               sudo = True,
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-                              environment = {'no_proxy': u'c6401.ambari.apache.org'},
-                              not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
-                              )
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+    )
     self.assertResourceCalled('File', '/tmp/start_hiveserver2_script',
                               content = Template('startHiveserver2.sh.j2'),
                               mode = 0755,
@@ -442,10 +441,9 @@ class TestHiveServer(RMFTestCase):
                               path = ['/bin', '/usr/bin/'],
                               sudo = True,
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-                              environment = {'no_proxy': u'c6401.ambari.apache.org'},
-                              not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
-                              )
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+    )
     self.assertResourceCalled('File', '/tmp/start_hiveserver2_script',
                               content = Template('startHiveserver2.sh.j2'),
                               mode = 0755,

ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py

@@ -30,11 +30,11 @@ class TestHookBeforeInstall(RMFTestCase):
                        command="hook",
                        config_file="default.json"
     )
-    self.assertResourceCalled('Execute', 'mkdir -p /tmp/AMBARI-artifacts/;     curl -kf -x "" --retry 10     http://c6401.ambari.apache.org:8080/resources//UnlimitedJCEPolicyJDK7.zip -o /tmp/AMBARI-artifacts//UnlimitedJCEPolicyJDK7.zip',
-        environment = {'no_proxy': 'c6401.ambari.apache.org'},
-        not_if = 'test -e /tmp/AMBARI-artifacts//UnlimitedJCEPolicyJDK7.zip',
-        ignore_failures = True,
-        path = ['/bin', '/usr/bin/'],
+    self.assertResourceCalled('Directory', '/tmp/AMBARI-artifacts/',
+        recursive = True,
+    )
+    self.assertResourceCalled('File', '/tmp/AMBARI-artifacts//UnlimitedJCEPolicyJDK7.zip',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//UnlimitedJCEPolicyJDK7.zip'),
     )
     self.assertResourceCalled('Group', 'hadoop',
         ignore_failures = False,

ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py

@@ -18,9 +18,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
-from stacks.utils.RMFTestCase import *
 from mock.mock import MagicMock, call, patch
 from resource_management import *
+from stacks.utils.RMFTestCase import *
 
 @patch.object(Hook, "run_custom_hook", new = MagicMock())
 class TestHookBeforeInstall(RMFTestCase):
@@ -40,10 +40,11 @@ class TestHookBeforeInstall(RMFTestCase):
     )
     self.assertResourceCalled('Package', 'unzip',)
     self.assertResourceCalled('Package', 'curl',)
-    self.assertResourceCalled('Execute', 'mkdir -p /tmp/AMBARI-artifacts/ ;   curl -kf -x \"\"   --retry 10 http://c6401.ambari.apache.org:8080/resources//jdk-7u67-linux-x64.tar.gz -o /tmp/AMBARI-artifacts//jdk-7u67-linux-x64.tar.gz',
-        not_if = 'test -e /usr/jdk64/jdk1.7.0_45/bin/java',
-        path = ['/bin', '/usr/bin/'],
-        environment = {'no_proxy': 'c6401.ambari.apache.org'},
+    self.assertResourceCalled('Directory', '/tmp/AMBARI-artifacts/',
+        recursive = True,
+    )
+    self.assertResourceCalled('File', '/tmp/AMBARI-artifacts//jdk-7u67-linux-x64.tar.gz',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//jdk-7u67-linux-x64.tar.gz'),
     )
     self.assertResourceCalled('Directory', '/usr/jdk64',)
     self.assertResourceCalled('Execute', ('chmod', 'a+x', u'/usr/jdk64'),

ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py

@@ -193,10 +193,9 @@ class TestHiveMetastore(RMFTestCase):
                               path = ['/bin', '/usr/bin/'],
                               sudo = True,
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-                              environment = {'no_proxy': u'c6401.ambari.apache.org'},
-                              not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
-                              )
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+    )
     self.assertResourceCalled('File', '/tmp/start_metastore_script',
                               content = StaticFile('startMetastore.sh'),
                               mode = 0755,
@@ -271,10 +270,9 @@ class TestHiveMetastore(RMFTestCase):
                               path = ['/bin', '/usr/bin/'],
                               sudo = True,
                               )
-    self.assertResourceCalled('Execute', '/bin/sh -c \'cd /usr/lib/ambari-agent/ && curl -kf -x "" --retry 5 http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar -o DBConnectionVerification.jar\'',
-                              environment = {'no_proxy': u'c6401.ambari.apache.org'},
-                              not_if = '[ -f /usr/lib/ambari-agent/DBConnectionVerification.jar ]',
-                              )
+    self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+        content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+    )
     self.assertResourceCalled('File', '/tmp/start_metastore_script',
                               content = StaticFile('startMetastore.sh'),
                               mode = 0755,

ambari-server/src/test/python/stacks/utils/RMFTestCase.py

@@ -17,7 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
-__all__ = ["RMFTestCase", "Template", "StaticFile", "InlineTemplate", "UnknownConfigurationMock", "FunctionMock",
+__all__ = ["RMFTestCase", "Template", "StaticFile", "InlineTemplate", "DownloadSource", "UnknownConfigurationMock", "FunctionMock",
            "CallFunctionMock"]
 
 from unittest import TestCase
@@ -244,7 +244,14 @@ def InlineTemplate(name, **kwargs):
   with RMFTestCase.env:
     from resource_management.core.source import InlineTemplate
     return InlineTemplate(name, **kwargs)
-
+  
+class DownloadSource():
+  def __init__(self, name, **kwargs):
+    self.name = name
+  
+  def __eq__(self, other):
+    from resource_management.core.source import DownloadSource
+    return isinstance(other, DownloadSource) and self.name == other.name
 
 class UnknownConfigurationMock():
   def __eq__(self, other):