AMBARI-10163. Enable kerberos user auth for Accumulo in 2.3 stack (Billie Rinaldi via smohanty)

ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-site.xml

@@ -40,32 +40,6 @@
     <description>Timeout for zookeeper connections.</description>
   </property>
 
-  <property>
-    <name>general.kerberos.principal</name>
-    <value>*</value>
-    <description>Name of the kerberos principal to use.</description>
-  </property>
-
-  <property>
-    <name>general.kerberos.keytab</name>
-    <value>*</value>
-    <description>Path to the kerberos keytab to use.</description>
-  </property>
-
-  <property>
-    <name>instance.rpc.ssl.enabled</name>
-    <value>false</value>
-    <description>Controls whether or not SSL is enabled for
-      Accumulo RPC.</description>
-  </property>
-
-  <property>
-    <name>instance.rpc.ssl.clientAuth</name>
-    <value>false</value>
-    <description>Controls whether or not client authentication is required
-      when SSL is enabled for Accumulo.</description>
-  </property>
-
   <property>
     <name>tserver.memory.maps.max</name>
     <value>80M</value>

ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/kerberos.json

@@ -58,7 +58,7 @@
           "name": "ACCUMULO_MASTER",
           "identities": [
             {
-              "name": "/accumulo_service"
+              "name": "./accumulo_service"
             }
           ]
         },
@@ -66,7 +66,7 @@
           "name": "ACCUMULO_TSERVER",
           "identities": [
             {
-              "name": "/accumulo_service"
+              "name": "./accumulo_service"
             }
           ]
         },
@@ -74,7 +74,7 @@
           "name": "ACCUMULO_MONITOR",
           "identities": [
             {
-              "name": "/accumulo_service"
+              "name": "./accumulo_service"
             }
           ]
         },
@@ -82,7 +82,7 @@
           "name": "ACCUMULO_GC",
           "identities": [
             {
-              "name": "/accumulo_service"
+              "name": "./accumulo_service"
             }
           ]
         }

ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_configuration.py

@@ -26,7 +26,7 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or '
 
   # create the conf directory
   Directory( params.conf_dir,
-      mode=0750,
+      mode=0755,
       owner = params.accumulo_user,
       group = params.user_group,
       recursive = True
@@ -107,6 +107,8 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or '
   configs = {}
   configs["instance.name"] = params.instance_name
   configs["instance.zookeeper.host"] = params.config['configurations']['accumulo-site']['instance.zookeeper.host']
+  if 'instance.rpc.sasl.enabled' in params.config['configurations']['accumulo-site']:
+    configs["instance.rpc.sasl.enabled"] = params.config['configurations']['accumulo-site']['instance.rpc.sasl.enabled']
   PropertiesFile(format("{dest_conf_dir}/client.conf"),
                  properties = configs,
                  owner = params.accumulo_user,
@@ -158,16 +160,10 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or '
                          mode=0700
     )
     params.HdfsDirectory(None, action="create")
-    passfile = format("{params.exec_tmp_dir}/pass")
-    try:
-      File(passfile,
-           mode=0600,
-           group=params.user_group,
-           owner=params.accumulo_user,
-           content=InlineTemplate('{{root_password}}\n'
-                                  '{{root_password}}\n')
-      )
-      Execute( format("cat {passfile} | {params.daemon_script} init "
+    if params.security_enabled and params.has_secure_user_auth:
+      Execute( format("{params.kinit_cmd} "
+                      "{params.daemon_script} init "
+                      "--user {params.accumulo_principal_name} "
                       "--instance-name {params.instance_name} "
                       "--clear-instance-name "
                       ">{params.log_dir}/accumulo-init.out "
@@ -178,12 +174,109 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or '
                                      "{params.instance_volumes}"),
                               params.accumulo_user),
                user=params.accumulo_user)
-    finally:
-      os.remove(passfile)
+    else:
+      passfile = format("{params.exec_tmp_dir}/pass")
+      try:
+        File(passfile,
+             mode=0600,
+             group=params.user_group,
+             owner=params.accumulo_user,
+             content=InlineTemplate('{{root_password}}\n'
+                                    '{{root_password}}\n')
+        )
+        Execute( format("cat {passfile} | {params.daemon_script} init "
+                        "--instance-name {params.instance_name} "
+                        "--clear-instance-name "
+                        ">{params.log_dir}/accumulo-init.out "
+                        "2>{params.log_dir}/accumulo-init.err"),
+                 not_if=as_user(format("{params.kinit_cmd} "
+                                       "{params.hadoop_bin_dir}/hadoop --config "
+                                       "{params.hadoop_conf_dir} fs -stat "
+                                       "{params.instance_volumes}"),
+                                params.accumulo_user),
+                 user=params.accumulo_user)
+      finally:
+        os.remove(passfile)
 
   if name == 'tracer':
+    if params.security_enabled and params.has_secure_user_auth:
+      Execute( format("{params.kinit_cmd} "
+                      "{params.daemon_script} init --reset-security "
+                      "--user {params.accumulo_principal_name} "
+                      "--password NA "
+                      ">{params.log_dir}/accumulo-reset.out "
+                      "2>{params.log_dir}/accumulo-reset.err"),
+               not_if=as_user(format("{params.kinit_cmd} "
+                                     "{params.daemon_script} shell -e "
+                                     "\"userpermissions -u "
+                                     "{params.accumulo_principal_name}\" | "
+                                     "grep System.CREATE_TABLE"),
+                              params.accumulo_user),
+               user=params.accumulo_user)
+      create_user(params.smokeuser_principal, params.smoke_test_password)
+    else:
+      # do not try to reset security in nonsecure mode, for now
+      # Execute( format("{params.daemon_script} init --reset-security "
+      #                 "--user root "
+      #                 ">{params.log_dir}/accumulo-reset.out "
+      #                 "2>{params.log_dir}/accumulo-reset.err"),
+      #          not_if=as_user(format("cat {rpassfile} | "
+      #                                "{params.daemon_script} shell -e "
+      #                                "\"userpermissions -u root\" | "
+      #                                "grep System.CREATE_TABLE"),
+      #                         params.accumulo_user),
+      #          user=params.accumulo_user)
+      create_user(params.smoke_test_user, params.smoke_test_password)
     create_user(params.trace_user, params.trace_password)
-    create_user(params.smoke_test_user, params.smoke_test_password)
+    rpassfile = format("{params.exec_tmp_dir}/pass0")
+    cmdfile = format("{params.exec_tmp_dir}/resetcmds")
+    try:
+      File(cmdfile,
+           mode=0600,
+           group=params.user_group,
+           owner=params.accumulo_user,
+           content=InlineTemplate('grant -t trace -u {{trace_user}} Table.ALTER_TABLE\n'
+                                  'grant -t trace -u {{trace_user}} Table.READ\n'
+                                  'grant -t trace -u {{trace_user}} Table.WRITE\n')
+      )
+      if params.security_enabled and params.has_secure_user_auth:
+        Execute( format("{params.kinit_cmd} {params.daemon_script} shell -f "
+                        "{cmdfile}"),
+                 only_if=as_user(format("{params.kinit_cmd} "
+                                        "{params.daemon_script} shell "
+                                        "-e \"table trace\""),
+                                 params.accumulo_user),
+                 not_if=as_user(format("{params.kinit_cmd} "
+                                       "{params.daemon_script} shell "
+                                       "-e \"userpermissions -u "
+                                       "{params.trace_user} | "
+                                       "grep Table.READ | grep trace"),
+                                params.accumulo_user),
+                 user=params.accumulo_user)
+      else:
+        File(rpassfile,
+             mode=0600,
+             group=params.user_group,
+             owner=params.accumulo_user,
+             content=InlineTemplate('{{root_password}}\n')
+        )
+        Execute( format("cat {rpassfile} | {params.daemon_script} shell -f "
+                        "{cmdfile} -u root"),
+                 only_if=as_user(format("cat {rpassfile} | "
+                                       "{params.daemon_script} shell -u root "
+                                       "-e \"table trace\""),
+                                params.accumulo_user),
+                 not_if=as_user(format("cat {rpassfile} | "
+                                       "{params.daemon_script} shell -u root "
+                                       "-e \"userpermissions -u "
+                                       "{params.trace_user} | "
+                                       "grep Table.READ | grep trace"),
+                                params.accumulo_user),
+                 user=params.accumulo_user)
+    finally:
+      try_remove(rpassfile)
+      try_remove(cmdfile)
+
 
 def create_user(user, password):
   import params
@@ -198,27 +291,36 @@ def create_user(user, password):
          content=InlineTemplate(format("createuser {user}\n"
                                        "grant -s System.CREATE_TABLE -u {user}\n"))
     )
-    File(rpassfile,
-         mode=0600,
-         group=params.user_group,
-         owner=params.accumulo_user,
-         content=InlineTemplate('{{root_password}}\n')
-    )
-    File(passfile,
-         mode=0600,
-         group=params.user_group,
-         owner=params.accumulo_user,
-         content=InlineTemplate(format("{params.root_password}\n"
-                                       "{password}\n"
-                                       "{password}\n"))
-    )
-    Execute( format("cat {passfile} | {params.daemon_script} shell -u root "
-                    "-f {cmdfile}"),
-             not_if=as_user(format("cat {rpassfile} | "
-                                   "{params.daemon_script} shell -u root "
-                                   "-e \"userpermissions -u {user}\""),
-                            params.accumulo_user),
-             user=params.accumulo_user)
+    if params.security_enabled and params.has_secure_user_auth:
+      Execute( format("{params.kinit_cmd} {params.daemon_script} shell -f "
+                      "{cmdfile}"),
+               not_if=as_user(format("{params.kinit_cmd} "
+                                     "{params.daemon_script} shell "
+                                     "-e \"userpermissions -u {user}\""),
+                              params.accumulo_user),
+               user=params.accumulo_user)
+    else:
+      File(rpassfile,
+           mode=0600,
+           group=params.user_group,
+           owner=params.accumulo_user,
+           content=InlineTemplate('{{root_password}}\n')
+      )
+      File(passfile,
+           mode=0600,
+           group=params.user_group,
+           owner=params.accumulo_user,
+           content=InlineTemplate(format("{params.root_password}\n"
+                                         "{password}\n"
+                                         "{password}\n"))
+      )
+      Execute( format("cat {passfile} | {params.daemon_script} shell -u root "
+                      "-f {cmdfile}"),
+               not_if=as_user(format("cat {rpassfile} | "
+                                     "{params.daemon_script} shell -u root "
+                                     "-e \"userpermissions -u {user}\""),
+                              params.accumulo_user),
+               user=params.accumulo_user)
   finally:
     try_remove(rpassfile)
     try_remove(passfile)

ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py

@@ -19,6 +19,7 @@ limitations under the License.
 """
 
 from resource_management import *
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management.libraries.functions.default import default
 import status_params
 
@@ -26,6 +27,16 @@ import status_params
 config = Script.get_config()
 exec_tmp_dir = status_params.tmp_dir
 
+# security enabled
+security_enabled = status_params.security_enabled
+
+# hdp version
+stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
+has_secure_user_auth = True
+if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') == 0:
+  has_secure_user_auth = False
+
 # accumulo local directory structure
 log_dir = config['configurations']['accumulo-env']['accumulo_log_dir']
 conf_dir = status_params.conf_dir # "/etc/accumulo/conf"
@@ -86,11 +97,11 @@ info_num_logs = config['configurations']['accumulo-log4j']['info_num_logs']
 # metrics2 properties
 ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present
 ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_hosts[0]
-ams_collector_hosts = default("/clusterHostInfo/metric_collector_hosts", [])
+ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
 has_metric_collector = not len(ams_collector_hosts) == 0
 if has_metric_collector:
   metric_collector_host = ams_collector_hosts[0]
-  metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:8188")
+  metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
   if metric_collector_port and metric_collector_port.find(':') != -1:
     metric_collector_port = metric_collector_port.split(':')[1]
   pass
@@ -99,7 +110,7 @@ if has_metric_collector:
 accumulo_user_keytab = config['configurations']['accumulo-env']['accumulo_user_keytab']
 accumulo_principal_name = config['configurations']['accumulo-env']['accumulo_principal_name']
 
-security_enabled = status_params.security_enabled
+# kinit properties
 kinit_path_local = status_params.kinit_path_local
 if security_enabled:
   kinit_cmd = format("{kinit_path_local} -kt {accumulo_user_keytab} {accumulo_principal_name};")

ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/service_check.py

@@ -41,11 +41,16 @@ class AccumuloServiceCheck(Script):
                                   'scan\n'
                                   'deletetable -f testtable\n')
       )
-      Execute( format("{smokeuser_kinit_cmd} "
-                      "{client_script} shell -u {smoke_test_user} "
-                      "-p {smoke_test_password} -f {cmdfile}"),
-               timeout=30,
-               user=params.smoke_test_user)
+      if params.security_enabled and params.has_secure_user_auth:
+        Execute( format("{smokeuser_kinit_cmd} "
+                        "{client_script} shell -f {cmdfile}"),
+                 timeout=30,
+                 user=params.smoke_test_user)
+      else:
+        Execute( format("{client_script} shell -u {smoke_test_user} "
+                        "-p {smoke_test_password} -f {cmdfile}"),
+                 timeout=30,
+                 user=params.smoke_test_user)
     finally:
       try_remove(cmdfile)
 

ambari-server/src/main/resources/stacks/HDP/2.3/services/ACCUMULO/configuration/accumulo-log4j.xml

@@ -0,0 +1,112 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+  <property>
+    <name>audit_log_level</name>
+    <value>OFF</value>
+    <description>Log level for audit logging</description>
+  </property>
+
+  <property>
+    <name>monitor_forwarding_log_level</name>
+    <value>WARN</value>
+    <description>Log level for logging forwarded to the Accumulo
+      Monitor</description>
+  </property>
+
+  <property>
+    <name>debug_log_size</name>
+    <value>1000M</value>
+    <description>Size of each debug rolling log file</description>
+  </property>
+
+  <property>
+    <name>debug_num_logs</name>
+    <value>10</value>
+    <description>Number of rolling debug log files to keep</description>
+  </property>
+
+  <property>
+    <name>info_log_size</name>
+    <value>1000M</value>
+    <description>Size of each info rolling log file</description>
+  </property>
+
+  <property>
+    <name>info_num_logs</name>
+    <value>10</value>
+    <description>Number of rolling info log files to keep</description>
+  </property>
+
+  <property>
+    <name>content</name>
+    <description>Custom log4j.properties</description>
+    <value>
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# default logging properties:
+#  by default, log everything at INFO or higher to the console
+log4j.rootLogger=INFO,A1
+
+# hide Jetty junk
+log4j.logger.org.mortbay.log=WARN,A1
+
+# hide "Got brand-new compressor" messages
+log4j.logger.org.apache.hadoop.io.compress=WARN,A1
+log4j.logger.org.apache.accumulo.core.file.rfile.bcfile.Compression=WARN,A1
+
+# hide junk from TestRandomDeletes
+log4j.logger.org.apache.accumulo.test.TestRandomDeletes=WARN,A1
+
+# hide junk from VFS
+log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN,A1
+
+# hide almost everything from zookeeper
+log4j.logger.org.apache.zookeeper=ERROR,A1
+
+# hide AUDIT messages in the shell, alternatively you could send them to a different logger
+log4j.logger.org.apache.accumulo.shell.Shell.audit=WARN,A1
+
+# Send most things to the console
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+log4j.appender.A1.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+
+    </value>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.3/services/ACCUMULO/kerberos.json

@@ -0,0 +1,145 @@
+{
+  "services": [
+    {
+      "name": "ACCUMULO",
+      "identities": [
+        {
+          "name": "accumulo",
+          "principal": {
+            "value": "${accumulo-env/accumulo_user}@${realm}",
+            "type" : "user",
+            "configuration": "accumulo-env/accumulo_principal_name",
+            "local_username": "${accumulo-env/accumulo_user}"
+          },
+          "keytab": {
+            "file": "${keytab_dir}/accumulo.headless.keytab",
+            "owner": {
+              "name": "${accumulo-env/accumulo_user}",
+              "access": "r"
+            },
+            "group": {
+              "name": "${cluster-env/user_group}",
+              "access": "r"
+            },
+            "configuration": "accumulo-env/accumulo_user_keytab"
+          }
+        },
+        {
+          "name": "accumulo_service",
+          "principal": {
+            "value": "${accumulo-env/accumulo_user}/_HOST@${realm}",
+            "type" : "service",
+            "configuration": "accumulo-site/general.kerberos.principal",
+            "local_username": "${accumulo-env/accumulo_user}"
+          },
+          "keytab": {
+            "file": "${keytab_dir}/accumulo.service.keytab",
+            "owner": {
+              "name": "${accumulo-env/accumulo_user}",
+              "access": "r"
+            },
+            "group": {
+              "name": "${cluster-env/user_group}",
+              "access": ""
+            },
+            "configuration": "accumulo-site/general.kerberos.keytab"
+          }
+        },
+        {
+          "name": "accumulo_tracer",
+          "principal": {
+            "value": "tracer@${realm}",
+            "type" : "user",
+            "configuration": "accumulo-site/trace.user",
+            "local_username": "${accumulo-env/accumulo_user}"
+          },
+          "keytab": {
+            "file": "${keytab_dir}/accumulo-tracer.headless.keytab",
+            "owner": {
+              "name": "${accumulo-env/accumulo_user}",
+              "access": "r"
+            },
+            "group": {
+              "name": "${cluster-env/user_group}",
+              "access": ""
+            },
+            "configuration": "accumulo-site/trace.token.property.keytab"
+          }
+        },
+        {
+          "name": "/hdfs"
+        },
+        {
+          "name": "/smokeuser"
+        }
+      ],
+      "configurations": [
+        {
+          "accumulo-site": {
+            "instance.rpc.sasl.enabled": "true",
+            "instance.security.authenticator": "org.apache.accumulo.server.security.handler.KerberosAuthenticator",
+            "instance.security.authorizor": "org.apache.accumulo.server.security.handler.KerberosAuthorizor",
+            "instance.security.permissionHandler": "org.apache.accumulo.server.security.handler.KerberosPermissionHandler",
+            "trace.token.type": "org.apache.accumulo.core.client.security.tokens.KerberosToken",
+            "general.delegation.token.lifetime": "7d",
+            "general.delegation.token.update.interval": "1d"
+          }
+        }
+      ],
+      "components": [
+        {
+          "name": "ACCUMULO_MASTER",
+          "identities": [
+            {
+              "name": "./accumulo_service"
+            }
+          ]
+        },
+        {
+          "name": "ACCUMULO_TSERVER",
+          "identities": [
+            {
+              "name": "./accumulo_service"
+            }
+          ]
+        },
+        {
+          "name": "ACCUMULO_MONITOR",
+          "identities": [
+            {
+              "name": "./accumulo_service"
+            },
+            {
+              "name": "./accumulo_tracer"
+            }
+          ]
+        },
+        {
+          "name": "ACCUMULO_GC",
+          "identities": [
+            {
+              "name": "./accumulo_service"
+            }
+          ]
+        },
+        {
+          "name": "ACCUMULO_TRACER",
+          "identities": [
+            {
+              "name": "./accumulo_tracer"
+            }
+          ]
+        },
+        {
+          "name": "ACCUMULO_CLIENT",
+          "identities": [
+            {
+              "name": "./accumulo"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
+