AMBARI-10884. Addition of security to Atlas service definition (Jon Maron via smohanty)

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/alerts.json

@@ -37,10 +37,13 @@
         "source": {
           "type": "WEB",
           "uri": {
-            "http": "{{metadata-env/metadata_port}}",
+            "http": "{{hostname}}:{{metadata-env/metadata_port}}",
+            "https": "{{hostname}}:21443",
+            "https_property": "{{application-properties/enableTLS}}",
+            "https_property_value": "true",
             "default_port": 21000,
-            "kerberos_keytab": "{{metadata-runtime.properties/*.metadata.http.authentication.keytab}}",
-            "kerberos_principal": "{{metadata-runtime.properties/*.metadata.http.authentication.principal}}"
+            "kerberos_keytab": "{{application-properties/http_authentication_kerberos_keytab}}",
+            "kerberos_principal": "{{application-properties/http_authentication_kerberos_principal}}"
           },
           "reporting": {
             "ok": {

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/kerberos.json

@@ -2,30 +2,28 @@
   "services": [
     {
       "name": "ATLAS",
-      "identities": [
-        {
-          "name": "/spnego"
-        }
-      ],
       "configurations": [
         {
-          "metadata-runtime.properties": {
-            "*.metadata.authentication.type": "kerberos",
-            "*.metadata.http.authentication.type": "kerberos",
-            "*.dfs.namenode.kerberos.principal": "nn/_HOST@${realm}"
+          "application-properties": {
+            "authentication_method": "kerberos",
+            "http_authentication_enabled": "true",
+            "http_authentication_type": "kerberos"
           }
         }
       ],
+      "auth_to_local_properties" : [
+        "application-properties/http_authentication_kerberos_name_rules"
+      ],
       "components": [
         {
           "name": "ATLAS_SERVER",
           "identities": [
             {
-              "name": "metadata_server",
+              "name": "atlas",
               "principal": {
                 "value": "atlas/_HOST@${realm}",
                 "type" : "service",
-                "configuration": "application-properties/*.metadata.authentication.principal",
+                "configuration": "application-properties/authentication_principal",
                 "local_username" : "${metadata-env/metadata_user}"
               },
               "keytab": {
@@ -38,17 +36,17 @@
                   "name": "${cluster-env/user_group}",
                   "access": ""
                 },
-                "configuration": "application-properties/*.metadata.authentication.keytab"
+                "configuration": "application-properties/authentication_keytab"
               }
             },
             {
               "name": "/spnego",
               "principal": {
                 "value": "HTTP/_HOST@${realm}",
-                "configuration": "application-properties/*.metadata.http.authentication.principal"
+                "configuration": "application-properties/http_authentication_kerberos_principal"
               },
               "keytab": {
-                "configuration": "application-properties/*.metadata.http.authentication.keytab"
+                "configuration": "application-properties/http_authentication_kerberos_keytab"
               }
             }
           ]

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py

@@ -17,6 +17,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
+import os
+import shutil
 from resource_management import Directory, Fail, Logger, File, \
     InlineTemplate, StaticFile
 from resource_management.libraries.functions import format
@@ -65,6 +67,13 @@ def metadata():
               recursive=True
     )
 
+    metadata_war_file = format('{params.metadata_home}/server/webapp/metadata.war')
+    if not os.path.isfile(metadata_war_file):
+        raise Fail("Unable to copy {0} because it does not exist".format(metadata_war_file))
+
+    Logger.info("Copying {0} to {1}".format(metadata_war_file, params.expanded_war_dir))
+    shutil.copy2(metadata_war_file, params.expanded_war_dir)
+
     File(format('{conf_dir}/application.properties'),
          content=InlineTemplate(params.application_properties_content),
          mode=0644,

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py

@@ -16,13 +16,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
-import os
-import shutil
-
 from metadata import metadata
-from resource_management import Execute, check_process_status, Script, Fail, \
-  Logger
+from resource_management import Execute, check_process_status, Script
 from resource_management.libraries.functions import format
+from resource_management.libraries.functions.security_commons import build_expectations, \
+  get_params_from_filesystem, validate_security_config_properties, \
+  FILE_TYPE_PROPERTIES
 
 class MetadataServer(Script):
 
@@ -31,7 +30,6 @@ class MetadataServer(Script):
 
   def install(self, env):
     self.install_packages(env)
-    self.configure(env)
 
   def configure(self, env):
     import params
@@ -46,13 +44,7 @@ class MetadataServer(Script):
   def start(self, env, rolling_restart=False):
     import params
     env.set_params(params)
-
-    metadata_war_file = format('{params.metadata_home}/server/webapp/metadata.war')
-    if not os.path.isfile(metadata_war_file):
-      raise Fail("Unable to copy {0} because it does not exist".format(metadata_war_file))
-
-    Logger.info("Copying {0} to {1}".format(metadata_war_file, params.expanded_war_dir))
-    shutil.copy2(metadata_war_file, params.expanded_war_dir)
+    self.configure(env)
 
     daemon_cmd = format('source {params.conf_dir}/metadata-env.sh ; {params.metadata_start_script} --port {params.metadata_port}')
     no_op_test = format('ls {params.pid_file} >/dev/null 2>&1 && ps -p `cat {params.pid_file}` >/dev/null 2>&1')
@@ -76,5 +68,60 @@ class MetadataServer(Script):
     env.set_params(status_params)
     check_process_status(status_params.pid_file)
 
+  def security_status(self, env):
+    import status_params
+
+    env.set_params(status_params)
+
+    props_value_check = {'metadata.authentication.method': 'kerberos',
+                         'metadata.http.authentication.enabled': 'true',
+                         'metadata.http.authentication.type': 'kerberos'}
+    props_empty_check = ['metadata.authentication.principal',
+                         'metadata.authentication.keytab',
+                         'metadata.http.authentication.kerberos.principal',
+                         'metadata.http.authentication.kerberos.keytab']
+    props_read_check = ['metadata.authentication.keytab',
+                        'metadata.http.authentication.kerberos.keytab']
+    atlas_site_expectations = build_expectations('application-properties',
+                                                    props_value_check,
+                                                    props_empty_check,
+                                                    props_read_check)
+
+    atlas_expectations = {}
+    atlas_expectations.update(atlas_site_expectations)
+
+    security_params = get_params_from_filesystem(status_params.conf_dir,
+                                                 {'application.properties': FILE_TYPE_PROPERTIES})
+    result_issues = validate_security_config_properties(security_params, atlas_expectations)
+    if not result_issues:  # If all validations passed successfully
+      try:
+        # Double check the dict before calling execute
+        if ( 'application-properties' not in security_params
+             or 'metadata.authentication.keytab' not in security_params['application-properties']
+             or 'metadata.authentication.principal' not in security_params['application-properties']):
+          self.put_structured_out({"securityState": "UNSECURED"})
+          self.put_structured_out(
+            {"securityIssuesFound": "Atlas service keytab file or principal are not set property."})
+          return
+
+        if ( 'application-properties' not in security_params
+             or 'metadata.http.authentication.kerberos.keytab' not in security_params['application-properties']
+             or 'metadata.http.authentication.kerberos.principal' not in security_params['application-properties']):
+          self.put_structured_out({"securityState": "UNSECURED"})
+          self.put_structured_out(
+            {"securityIssuesFound": "HTTP Authentication keytab file or principal are not set property."})
+          return
+
+        self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+      except Exception as e:
+        self.put_structured_out({"securityState": "ERROR"})
+        self.put_structured_out({"securityStateErrorInfo": str(e)})
+    else:
+      issues = []
+      for cf in result_issues:
+        issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf]))
+      self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+      self.put_structured_out({"securityState": "UNSECURED"})
+
 if __name__ == "__main__":
   MetadataServer().execute()

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py

@@ -69,6 +69,9 @@ metadata_host = config['hostname']
 application_properties = config['configurations']['application-properties']
 
 for key, value in application_properties.iteritems():
+    #  fix the multi-line property
+    if (key == 'http_authentication_kerberos_name_rules'):
+        value = ' \\ \n'.join(value.splitlines())
     globals()[key] = value
 
 metadata_env_content = config['configurations']['metadata-env']['content']
@@ -77,4 +80,18 @@ application_properties_content = config['configurations']['application-propertie
 metadata_opts = config['configurations']['metadata-env']['metadata_opts']
 metadata_classpath = config['configurations']['metadata-env']['metadata_classpath']
 data_dir = config['configurations']['metadata-env']['metadata_data_dir']
-expanded_war_dir = os.environ['METADATA_EXPANDED_WEBAPP_DIR'] if 'METADATA_EXPANDED_WEBAPP_DIR' in os.environ else '/var/lib/atlas/server/webapp'
+expanded_war_dir = os.environ['METADATA_EXPANDED_WEBAPP_DIR'] if 'METADATA_EXPANDED_WEBAPP_DIR' in os.environ else '/var/lib/atlas/server/webapp'
+
+# smoke test
+smoke_test_user = config['configurations']['cluster-env']['smokeuser']
+smoke_test_password = 'smoke'
+smokeuser_principal =  config['configurations']['cluster-env']['smokeuser_principal_name']
+smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+
+kinit_path_local = status_params.kinit_path_local
+
+security_check_status_file = format('{log_dir}/security_check.status')
+if security_enabled:
+    smoke_cmd = format('curl --negotiate -u : -b ~/cookiejar.txt -c ~/cookiejar.txt -s -o /dev/null -w "%{{http_code}}" http://{metadata_host}:{metadata_port}/')
+else:
+    smoke_cmd = format('curl -s -o /dev/null -w "%{{http_code}}" http://{metadata_host}:{metadata_port}/')

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/service_check.py

@@ -16,42 +16,29 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
-import httplib
-import socket
-import time
-from resource_management import Script, Logger, ComponentIsNotRunning, Fail
+from resource_management import Script, Logger, ComponentIsNotRunning, \
+  Execute
 from resource_management.libraries.functions import format
 
 
 class AtlasServiceCheck(Script):
-  ATLAS_CONNECT_TRIES = 5
-  ATLAS_CONNECT_TIMEOUT = 10
 
   def service_check(self, env):
     import params
 
     env.set_params(params)
 
-    for i in xrange(0, self.ATLAS_CONNECT_TRIES):
-      try:
-        conn = httplib.HTTPConnection(params.metadata_host,
-                                      int(params.metadata_port))
-        conn.request("GET", format("http://{params.metadata_host}:{params.metadata_port}/"))
-      except (httplib.HTTPException, socket.error) as ex:
-        if i < self.ATLAS_CONNECT_TRIES - 1:
-          time.sleep(self.ATLAS_CONNECT_TIMEOUT)
-          Logger.info("Connection failed. Next retry in %s seconds."
-                      % (self.ATLAS_CONNECT_TIMEOUT))
-          continue
-        else:
-          raise Fail("Service check has failed.")
-
-    resp = conn.getresponse()
-    if resp.status == 200 :
+    if params.security_enabled:
+      Execute(format("{kinit_path_local} -kt {smokeuser_keytab} {smokeuser_principal}"),
+              user=params.metadata_user)
+
+    try:
+      Execute(params.smoke_cmd, user=params.metadata_user, tries = 5,
+              try_sleep = 10)
       Logger.info('Atlas server up and running')
-    else:
+    except:
       Logger.debug('Atlas server not running')
-      raise ComponentIsNotRunning()
+
 
 if __name__ == "__main__":
   AtlasServiceCheck().execute()

ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py

@@ -20,6 +20,7 @@ limitations under the License.
 import os
 from resource_management import Script
 from resource_management.libraries.functions import  get_kinit_path, format
+from resource_management.libraries.functions.default import default
 
 
 config = Script.get_config()
@@ -32,5 +33,5 @@ metadata_user = config['configurations']['metadata-env']['metadata_user']
 # Security related/required params
 hostname = config['hostname']
 security_enabled = config['configurations']['cluster-env']['security_enabled']
-kinit_path_local = get_kinit_path()
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
 tmp_dir = Script.get_tmp_dir()

ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py

@@ -28,7 +28,70 @@ class TestMetadataServer(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "ATLAS/0.1.0.2.3/package"
   STACK_VERSION = "2.3"
 
-  def test_configure_default(self):
+  def configureResourcesCalled(self):
+      self.assertResourceCalled('Directory', '/var/run/atlas',
+                                owner='atlas',
+                                group='hadoop',
+                                recursive=True,
+                                cd_access='a',
+                                mode=0755
+      )
+      self.assertResourceCalled('Directory', '/etc/atlas/conf',
+                                owner='atlas',
+                                group='hadoop',
+                                recursive=True,
+                                cd_access='a',
+                                mode=0755
+      )
+      self.assertResourceCalled('Directory', '/var/log/atlas',
+                                owner='atlas',
+                                group='hadoop',
+                                recursive=True,
+                                cd_access='a',
+                                mode=0755
+      )
+      self.assertResourceCalled('Directory', '/var/lib/atlas/data',
+                                owner='atlas',
+                                group='hadoop',
+                                recursive=True,
+                                cd_access='a',
+                                mode=0644
+      )
+      self.assertResourceCalled('Directory', '/var/lib/atlas/server/webapp',
+                                owner='atlas',
+                                group='hadoop',
+                                recursive=True,
+                                cd_access='a',
+                                mode=0644
+      )
+      self.assertResourceCalled('File',
+                                '/etc/atlas/conf/application.properties',
+                                content=InlineTemplate(
+                                    self.getConfig()['configurations'][
+                                        'application-properties']['content']),
+                                owner='atlas',
+                                group='hadoop',
+                                mode=0644,
+      )
+      self.assertResourceCalled('File', '/etc/atlas/conf/metadata-env.sh',
+                                content=InlineTemplate(
+                                    self.getConfig()['configurations'][
+                                        'metadata-env']['content']),
+                                owner='atlas',
+                                group='hadoop',
+                                mode=0755,
+      )
+      self.assertResourceCalled('File', '/etc/atlas/conf/log4j.xml',
+                                content=StaticFile('log4j.xml'),
+                                owner='atlas',
+                                group='hadoop',
+                                mode=0644,
+      )
+
+  @patch("shutil.copy2", new = MagicMock())
+  @patch("os.path.isfile")
+  def test_configure_default(self, isfile_mock):
+    isfile_mock.return_value = True
 
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py",
                        classname = "MetadataServer",
@@ -38,59 +101,7 @@ class TestMetadataServer(RMFTestCase):
                        target = RMFTestCase.TARGET_COMMON_SERVICES
     )
 
-    self.assertResourceCalled('Directory', '/var/run/atlas',
-                              owner = 'atlas',
-                              group = 'hadoop',
-                              recursive = True,
-                              cd_access = 'a',
-                              mode=0755
-    )
-    self.assertResourceCalled('Directory', '/etc/atlas/conf',
-                              owner = 'atlas',
-                              group = 'hadoop',
-                              recursive = True,
-                              cd_access = 'a',
-                              mode=0755
-    )
-    self.assertResourceCalled('Directory', '/var/log/atlas',
-                              owner = 'atlas',
-                              group = 'hadoop',
-                              recursive = True,
-                              cd_access = 'a',
-                              mode=0755
-    )
-    self.assertResourceCalled('Directory', '/var/lib/atlas/data',
-                              owner = 'atlas',
-                              group = 'hadoop',
-                              recursive = True,
-                              cd_access = 'a',
-                              mode=0644
-    )
-    self.assertResourceCalled('Directory', '/var/lib/atlas/server/webapp',
-                              owner = 'atlas',
-                              group = 'hadoop',
-                              recursive = True,
-                              cd_access = 'a',
-                              mode=0644
-    )
-    self.assertResourceCalled('File', '/etc/atlas/conf/application.properties',
-                              content = InlineTemplate(self.getConfig()['configurations']['application-properties']['content']),
-                              owner = 'atlas',
-                              group = 'hadoop',
-                              mode = 0644,
-                              )
-    self.assertResourceCalled('File', '/etc/atlas/conf/metadata-env.sh',
-                              content = InlineTemplate(self.getConfig()['configurations']['metadata-env']['content']),
-                              owner = 'atlas',
-                              group = 'hadoop',
-                              mode = 0755,
-                              )
-    self.assertResourceCalled('File', '/etc/atlas/conf/log4j.xml',
-                              content = StaticFile('log4j.xml'),
-                              owner = 'atlas',
-                              group = 'hadoop',
-                              mode = 0644,
-                              )
+    self.configureResourcesCalled()
     self.assertNoMoreResources()
 
   @patch("shutil.copy2", new = MagicMock())
@@ -105,6 +116,7 @@ class TestMetadataServer(RMFTestCase):
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES
     )
+    self.configureResourcesCalled()
     self.assertResourceCalled('Execute', 'source /etc/atlas/conf/metadata-env.sh ; /usr/hdp/current/atlas-server/bin/metadata_start.py --port 21000',
                               not_if = 'ls /var/run/atlas/metadata.pid >/dev/null 2>&1 && ps -p `cat /var/run/atlas/metadata.pid` >/dev/null 2>&1',
                               user = 'atlas',