AMBARI-6185. Secure cluster: JCE policy files not distributed on non-client hosts. (aonishuk)

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-ANY/scripts/hook.py

@@ -0,0 +1,33 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from shared_initialization import *
+
+class BeforeAnyHook(Hook):
+
+  def hook(self, env):
+    import params
+    env.set_params(params)
+    
+    setup_jce()
+
+if __name__ == "__main__":
+  BeforeAnyHook().execute()
+

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-ANY/scripts/params.py

@@ -0,0 +1,33 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+
+_authentication = config['configurations']['core-site']['hadoop.security.authentication']
+security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
+
+artifact_dir = "/tmp/HDP-artifacts/"
+jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user
+jce_location = config['hostLevelParams']['jdk_location']
+jdk_name = default("/hostLevelParams/jdk_name", None)
+java_home = config['hostLevelParams']['java_home']
+
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-ANY/scripts/shared_initialization.py

@@ -0,0 +1,58 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management import *
+
+
+
+def setup_jce():
+  import params
+  
+  if not params.jdk_name:
+    return
+  
+  environment = {
+    "no_proxy": format("{ambari_server_hostname}")
+  }
+  
+  if params.jce_policy_zip is not None:
+    jce_curl_target = format("{artifact_dir}/{jce_policy_zip}")
+    download_jce = format("mkdir -p {artifact_dir}; \
+    curl -kf --retry 10 \
+    {jce_location}/{jce_policy_zip} -o {jce_curl_target}")
+    Execute( download_jce,
+             path = ["/bin","/usr/bin/"],
+             not_if =format("test -e {jce_curl_target}"),
+             ignore_failures = True,
+             environment = environment
+    )
+  elif params.security_enabled:
+    # Something weird is happening
+    raise Fail("Security is enabled, but JCE policy zip is not specified.")
+  
+  if params.security_enabled:
+    security_dir = format("{java_home}/jre/lib/security")
+    extract_cmd = format("rm -f local_policy.jar; rm -f US_export_policy.jar; unzip -o -j -q {jce_curl_target}")
+    Execute(extract_cmd,
+            only_if = format("test -e {security_dir} && test -f {jce_curl_target}"),
+            cwd  = security_dir,
+            path = ['/bin/','/usr/bin']
+    )

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-INSTALL/scripts/hook.py

@@ -22,17 +22,18 @@ from resource_management import *
 from shared_initialization import *
 from repo_initialization import install_repos
 
-#TODO this must be "CONFIGURE" hook when CONFIGURE command will be implemented
-class BeforeConfigureHook(Hook):
+class BeforeInstallHook(Hook):
 
   def hook(self, env):
     import params
-
+    
+    self.run_custom_hook('before-ANY')
     env.set_params(params)
+    
     install_repos()
     setup_java()
     setup_users()
     install_packages()
 
 if __name__ == "__main__":
-  BeforeConfigureHook().execute()
+  BeforeInstallHook().execute()

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-INSTALL/scripts/shared_initialization.py

@@ -170,29 +170,5 @@ def setup_java():
           not_if = format("test -e {java_exec}")
   )
 
-  if params.jce_policy_zip is not None:
-    jce_curl_target = format("{artifact_dir}/{jce_policy_zip}")
-    download_jce = format("mkdir -p {artifact_dir}; curl -kf "
-                          "--retry 10 {jce_location}/{jce_policy_zip} "
-                          "-o {jce_curl_target}")
-    Execute( download_jce,
-             path = ["/bin","/usr/bin/"],
-             not_if =format("test -e {jce_curl_target}"),
-             environment = environment,
-             ignore_failures = True
-    )
-  elif params.security_enabled:
-    # Something weird is happening
-    raise Fail("Security is enabled, but JCE policy zip is not specified.")
-
-  if params.security_enabled:
-    security_dir = format("{java_home}/jre/lib/security")
-    extract_cmd = format("rm -f local_policy.jar; rm -f US_export_policy.jar; unzip -o -j -q {jce_curl_target}")
-    Execute(extract_cmd,
-            only_if = format("test -e {security_dir} && test -f {jce_curl_target}"),
-            cwd  = security_dir,
-            path = ['/bin/','/usr/bin']
-    )
-
 def install_packages():
   Package(['unzip'])

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-RESTART/scripts/hook.py

@@ -19,11 +19,11 @@ limitations under the License.
 
 from resource_management import *
 
-class BeforeConfigureHook(Hook):
+class BeforeRestartHook(Hook):
 
   def hook(self, env):
     self.run_custom_hook('before-START')
 
 if __name__ == "__main__":
-  BeforeConfigureHook().execute()
+  BeforeRestartHook().execute()
 

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/hook.py

@@ -21,18 +21,19 @@ import sys
 from resource_management import *
 from shared_initialization import *
 
-#TODO this must be "CONFIGURE" hook when CONFIGURE command will be implemented
-class BeforeConfigureHook(Hook):
+class BeforeStartHook(Hook):
 
   def hook(self, env):
     import params
 
-    env.set_params(params)
+    self.run_custom_hook('before-ANY')
     self.run_custom_hook('after-INSTALL')
+    env.set_params(params)
+    
     setup_hadoop()
     setup_database()
     setup_configs()
     create_javahome_symlink()
 
 if __name__ == "__main__":
-  BeforeConfigureHook().execute()
+  BeforeStartHook().execute()

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/hook.py

@@ -0,0 +1,33 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from shared_initialization import *
+
+class BeforeAnyHook(Hook):
+
+  def hook(self, env):
+    import params
+    env.set_params(params)
+    
+    setup_jce()
+
+if __name__ == "__main__":
+  BeforeAnyHook().execute()
+

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py

@@ -0,0 +1,33 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+
+_authentication = config['configurations']['core-site']['hadoop.security.authentication']
+security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
+
+artifact_dir = "/tmp/HDP-artifacts/"
+jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user
+jce_location = config['hostLevelParams']['jdk_location']
+jdk_name = default("/hostLevelParams/jdk_name", None)
+java_home = config['hostLevelParams']['java_home']
+
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py

@@ -0,0 +1,58 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management import *
+
+
+
+def setup_jce():
+  import params
+  
+  if not params.jdk_name:
+    return
+  
+  environment = {
+    "no_proxy": format("{ambari_server_hostname}")
+  }
+  
+  if params.jce_policy_zip is not None:
+    jce_curl_target = format("{artifact_dir}/{jce_policy_zip}")
+    download_jce = format("mkdir -p {artifact_dir}; \
+    curl -kf --retry 10 \
+    {jce_location}/{jce_policy_zip} -o {jce_curl_target}")
+    Execute( download_jce,
+             path = ["/bin","/usr/bin/"],
+             not_if =format("test -e {jce_curl_target}"),
+             ignore_failures = True,
+             environment = environment
+    )
+  elif params.security_enabled:
+    # Something weird is happening
+    raise Fail("Security is enabled, but JCE policy zip is not specified.")
+  
+  if params.security_enabled:
+    security_dir = format("{java_home}/jre/lib/security")
+    extract_cmd = format("rm -f local_policy.jar; rm -f US_export_policy.jar; unzip -o -j -q {jce_curl_target}")
+    Execute(extract_cmd,
+            only_if = format("test -e {security_dir} && test -f {jce_curl_target}"),
+            cwd  = security_dir,
+            path = ['/bin/','/usr/bin']
+    )

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/hook.py

@@ -21,17 +21,19 @@ import sys
 from resource_management import *
 from shared_initialization import *
 from repo_initialization import *
-#TODO this must be "CONFIGURE" hook when CONFIGURE command will be implemented
-class BeforeConfigureHook(Hook):
+
+class BeforeInstallHook(Hook):
 
   def hook(self, env):
     import params
 
+    self.run_custom_hook('before-ANY')
     env.set_params(params)
+    
     install_repos()
     install_packages()
     setup_java()
     setup_users()
 
 if __name__ == "__main__":
-  BeforeConfigureHook().execute()
+  BeforeInstallHook().execute()

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py

@@ -157,7 +157,7 @@ def set_uid(user, user_dirs):
        mode=0555)
   Execute(format("/tmp/changeUid.sh {user} {user_dirs} 2>/dev/null"),
           not_if = format("test $(id -u {user}) -gt 1000"))
-
+  
 def setup_java():
   """
   Installs jdk using specific params, that comes from ambari-server
@@ -192,29 +192,5 @@ def setup_java():
           not_if = format("test -e {java_exec}")
   )
 
-  if params.jce_policy_zip is not None:
-    jce_curl_target = format("{artifact_dir}/{jce_policy_zip}")
-    download_jce = format("mkdir -p {artifact_dir}; \
-    curl -kf --retry 10 \
-    {jce_location}/{jce_policy_zip} -o {jce_curl_target}")
-    Execute( download_jce,
-             path = ["/bin","/usr/bin/"],
-             not_if =format("test -e {jce_curl_target}"),
-             ignore_failures = True,
-             environment = environment
-    )
-  elif params.security_enabled:
-    # Something weird is happening
-    raise Fail("Security is enabled, but JCE policy zip is not specified.")
-
-  if params.security_enabled:
-    security_dir = format("{java_home}/jre/lib/security")
-    extract_cmd = format("rm -f local_policy.jar; rm -f US_export_policy.jar; unzip -o -j -q {jce_curl_target}")
-    Execute(extract_cmd,
-            only_if = format("test -e {security_dir} && test -f {jce_curl_target}"),
-            cwd  = security_dir,
-            path = ['/bin/','/usr/bin']
-    )
-
 def install_packages():
   Package(['unzip', 'curl'])

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-RESTART/scripts/hook.py

@@ -19,11 +19,11 @@ limitations under the License.
 
 from resource_management import *
 
-class BeforeConfigureHook(Hook):
+class BeforeRestartHook(Hook):
 
   def hook(self, env):
     self.run_custom_hook('before-START')
 
 if __name__ == "__main__":
-  BeforeConfigureHook().execute()
+  BeforeRestartHook().execute()
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/hook.py

@@ -21,17 +21,18 @@ import sys
 from resource_management import *
 from shared_initialization import *
 
-#TODO this must be "CONFIGURE" hook when CONFIGURE command will be implemented
-class BeforeConfigureHook(Hook):
+class BeforeStartHook(Hook):
 
   def hook(self, env):
     import params
 
-    env.set_params(params)
+    self.run_custom_hook('before-ANY')
     self.run_custom_hook('after-INSTALL')
+    env.set_params(params)
+
     setup_hadoop()
     setup_configs()
     create_javahome_symlink()
 
 if __name__ == "__main__":
-  BeforeConfigureHook().execute()
+  BeforeStartHook().execute()

ambari-server/src/test/python/stacks/1.3.2/hooks/before-INSTALL/test_before_install.py

@@ -18,13 +18,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
+from mock.mock import MagicMock, call, patch
+from resource_management import *
 from stacks.utils.RMFTestCase import *
-from resource_management.core.exceptions import *
 
+@patch.object(Hook, "run_custom_hook", new = MagicMock())
 class TestHookBeforeInstall(RMFTestCase):
   def test_hook_default(self):
     self.executeScript("1.3.2/hooks/before-INSTALL/scripts/hook.py",
-                       classname="BeforeConfigureHook",
+                       classname="BeforeInstallHook",
                        command="hook",
                        config_file="default.json"
     )
@@ -45,12 +47,6 @@ class TestHookBeforeInstall(RMFTestCase):
         not_if = 'test -e /usr/jdk64/jdk1.7.0_45/bin/java',
         path = ['/bin', '/usr/bin/'],
     )
-    self.assertResourceCalled('Execute', 'mkdir -p /tmp/HDP-artifacts/; curl -kf --retry 10 http://c6401.ambari.apache.org:8080/resources//UnlimitedJCEPolicyJDK7.zip -o /tmp/HDP-artifacts//UnlimitedJCEPolicyJDK7.zip',
-        not_if = 'test -e /tmp/HDP-artifacts//UnlimitedJCEPolicyJDK7.zip',
-        ignore_failures = True,
-        path = ['/bin', '/usr/bin/'],
-        environment = {'no_proxy': 'c6401.ambari.apache.org'}
-    )
     self.assertResourceCalled('Group', 'hadoop',
         ignore_failures = False,
     )
@@ -138,18 +134,4 @@ class TestHookBeforeInstall(RMFTestCase):
         ignore_failures = False,
     )
     self.assertResourceCalled('Package', 'unzip',)
-    self.assertNoMoreResources()
-
-
-  def test_that_jce_is_required_in_secured_cluster(self):
-    try:
-      self.executeScript("1.3.2/hooks/before-INSTALL/scripts/hook.py",
-                         classname="BeforeConfigureHook",
-                         command="hook",
-                         config_file="secured_no_jce_name.json"
-      )
-      self.fail("Should throw an exception")
-    except Fail:
-      pass  # Expected
-
-
+    self.assertNoMoreResources()

ambari-server/src/test/python/stacks/1.3.2/hooks/before-START/test_before_start.py

@@ -27,7 +27,7 @@ from stacks.utils.RMFTestCase import *
 class TestHookBeforeStart(RMFTestCase):
   def test_hook_default(self, mockHook):
     self.executeScript("1.3.2/hooks/before-START/scripts/hook.py",
-                       classname="BeforeConfigureHook",
+                       classname="BeforeStartHook",
                        command="hook",
                        config_file="default.json"
     )
@@ -85,7 +85,7 @@ class TestHookBeforeStart(RMFTestCase):
 
   def test_hook_secured(self, mockHook):
     self.executeScript("1.3.2/hooks/before-START/scripts/hook.py",
-                       classname="BeforeConfigureHook",
+                       classname="BeforeStartHook",
                        command="hook",
                        config_file="secured.json"
     )
@@ -136,3 +136,14 @@ class TestHookBeforeStart(RMFTestCase):
                               group = 'hadoop',
                               )
     self.assertNoMoreResources()
+    
+def test_that_jce_is_required_in_secured_cluster(self):
+  try:
+    self.executeScript("2.0.6/hooks/before-START/scripts/hook.py",
+                       classname="BeforeStartHook",
+                       command="hook",
+                       config_file="secured_no_jce_name.json"
+    )
+    self.fail("Should throw an exception")
+  except Fail:
+    pass  # Expected

ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py

@@ -18,13 +18,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
+from mock.mock import MagicMock, call, patch
+from resource_management import *
 from stacks.utils.RMFTestCase import *
-from resource_management.core.exceptions import *
 
+@patch.object(Hook, "run_custom_hook", new = MagicMock())
 class TestHookBeforeInstall(RMFTestCase):
   def test_hook_default(self):
     self.executeScript("2.0.6/hooks/before-INSTALL/scripts/hook.py",
-                       classname="BeforeConfigureHook",
+                       classname="BeforeInstallHook",
                        command="hook",
                        config_file="default.json"
     )
@@ -47,12 +49,6 @@ class TestHookBeforeInstall(RMFTestCase):
         not_if = 'test -e /usr/jdk64/jdk1.7.0_45/bin/java',
         path = ['/bin', '/usr/bin/'],
     )
-    self.assertResourceCalled('Execute', 'mkdir -p /tmp/HDP-artifacts/;     curl -kf --retry 10     http://c6401.ambari.apache.org:8080/resources//UnlimitedJCEPolicyJDK7.zip -o /tmp/HDP-artifacts//UnlimitedJCEPolicyJDK7.zip',
-        not_if = 'test -e /tmp/HDP-artifacts//UnlimitedJCEPolicyJDK7.zip',
-        ignore_failures = True,
-        path = ['/bin', '/usr/bin/'],
-        environment = {'no_proxy': 'c6401.ambari.apache.org'},
-    )
     self.assertResourceCalled('Group', 'hadoop',
         ignore_failures = False,
     )
@@ -159,15 +155,3 @@ class TestHookBeforeInstall(RMFTestCase):
         groups = ['users'],
     )
     self.assertNoMoreResources()
-
-
-  def test_that_jce_is_required_in_secured_cluster(self):
-    try:
-      self.executeScript("2.0.6/hooks/before-INSTALL/scripts/hook.py",
-                         classname="BeforeConfigureHook",
-                         command="hook",
-                         config_file="secured_no_jce_name.json"
-      )
-      self.fail("Should throw an exception")
-    except Fail:
-      pass  # Expected

ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py

@@ -27,7 +27,7 @@ from stacks.utils.RMFTestCase import *
 class TestHookBeforeStart(RMFTestCase):
   def test_hook_default(self):
     self.executeScript("2.0.6/hooks/before-START/scripts/hook.py",
-                       classname="BeforeConfigureHook",
+                       classname="BeforeStartHook",
                        command="hook",
                        config_file="default.json"
     )
@@ -87,7 +87,7 @@ class TestHookBeforeStart(RMFTestCase):
 
   def test_hook_secured(self):
     self.executeScript("2.0.6/hooks/before-START/scripts/hook.py",
-                       classname="BeforeConfigureHook",
+                       classname="BeforeStartHook",
                        command="hook",
                        config_file="secured.json"
     )
@@ -144,3 +144,14 @@ class TestHookBeforeStart(RMFTestCase):
                               group = 'hadoop',
                               )
     self.assertNoMoreResources()
+    
+def test_that_jce_is_required_in_secured_cluster(self):
+  try:
+    self.executeScript("2.0.6/hooks/before-START/scripts/hook.py",
+                       classname="BeforeStartHook",
+                       command="hook",
+                       config_file="secured_no_jce_name.json"
+    )
+    self.fail("Should throw an exception")
+  except Fail:
+    pass  # Expected