AMBARI-16031. Create "/hadoop/llap/local" on each host and disk in Kerberized cluster for LLAP (alejandro)

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json

@@ -17,6 +17,11 @@
             "hive.server2.authentication": "KERBEROS"
           }
         },
+        {
+          "hive-interactive-site": {
+            "hive.llap.daemon.work.dirs": "/hadoop/llap/local"
+          }
+        },
         {
           "webhcat-site": {
             "templeton.kerberos.secret": "secret",

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml

@@ -91,6 +91,16 @@
              <!-- Used by decommission and recommission -->
              <masterComponent>RESOURCEMANAGER</masterComponent>
            </bulkCommands>
+          <customCommands>
+            <customCommand>
+              <name>CREATE_YARN_DIRECTORIES</name>
+              <commandScript>
+                <script>scripts/nodemanager.py</script>
+                <scriptType>PYTHON</scriptType>
+                <timeout>600</timeout>
+              </commandScript>
+            </customCommand>
+          </customCommands>
           <logs>
             <log>
               <logId>yarn_nodemanager</logId>

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py

@@ -30,7 +30,7 @@ from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.security_commons import build_expectations, \
   cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
   FILE_TYPE_XML
-from yarn import yarn
+from yarn import yarn, create_hive_llap_work_dir
 from service import service
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyImpl
@@ -90,6 +90,16 @@ class NodemanagerDefault(Nodemanager):
     env.set_params(status_params)
     check_process_status(status_params.nodemanager_pid_file)
 
+  def create_yarn_directories(self, env):
+    """
+    Custom command to Create Directories, e.g., needed by YARN Apps on Slider
+    """
+    Logger.info("Custom Command to Create Directories")
+    import params
+    env.set_params(params)
+    create_hive_llap_work_dir(params)
+
+
   def security_status(self, env):
     import status_params
     env.set_params(status_params)

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py

@@ -19,6 +19,7 @@ Ambari Agent
 
 """
 import os
+import re
 
 from resource_management.libraries.script.script import Script
 from resource_management.libraries.resources.hdfs_resource import HdfsResource
@@ -184,11 +185,39 @@ nm_address = config['configurations']['yarn-site']['yarn.nodemanager.address']
 if hostname and nm_address and nm_address.startswith("0.0.0.0:"):
   nm_address = nm_address.replace("0.0.0.0", hostname)
 
-nm_local_dirs = config['configurations']['yarn-site']['yarn.nodemanager.local-dirs']
-nm_log_dirs = config['configurations']['yarn-site']['yarn.nodemanager.log-dirs']
+# Initialize lists of work directories.
+nm_local_dirs = default("/configurations/yarn-site/yarn.nodemanager.local-dirs", "")
+nm_log_dirs = default("/configurations/yarn-site/yarn.nodemanager.log-dirs", "")
 
-nm_local_dirs_list = nm_local_dirs.split(',')
-nm_log_dirs_list = nm_log_dirs.split(',')
+# If Hive Interactive Server is present, then default value will be ${yarn.nodemanager.local-dirs}
+# During kerberization, kerberos.json changes the value to "/hadoop/llap/local".
+# This is a comma separated list of directories.
+hive_llap_work_dirs = default("/configurations/hive-interactive-site/hive.llap.daemon.work.dirs", "")
+
+# It can also be a dynamic variable, such as "${yarn.nodemanager.local-dirs}"
+p = re.compile(r"\$\{(.*?)\}")
+m = p.search(hive_llap_work_dirs)
+if m and len(m.groups()) == 1:
+  hive_llap_work_dirs = default("/configurations/yarn-site/" + m.group(1), "")
+
+nm_local_dirs_list = nm_local_dirs.split(",")
+nm_log_dirs_list = nm_log_dirs.split(",")
+hive_llap_work_dirs_list = hive_llap_work_dirs.split(",")
+
+
+def cleanse_list(l):
+  if l is None:
+    return None
+  l = [e.strip() for e in l if e.strip() != ""]
+  return l if len(l) > 0 else None
+
+nm_local_dirs_list = cleanse_list(nm_local_dirs_list)
+nm_log_dirs_list = cleanse_list(nm_log_dirs_list)
+hive_llap_work_dirs_list = cleanse_list(hive_llap_work_dirs_list)
+
+# Needed for LLAP to determine if should create Hive LLAP Work Dirs.
+hive_server_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', [])
+hive_server_interactive_host = hive_server_interactive_hosts[0] if len(hive_server_interactive_hosts) > 0 else None
 
 distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar"
 hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py

@@ -19,11 +19,25 @@ Ambari Agent
 
 """
 
-from resource_management import *
+# Python Imports
 import os
+
+# Ambari Common and Resource Management Imports
+from resource_management.libraries.script.script import Script
+from resource_management.core.resources.service import ServiceConfig
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import File
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.core.source import InlineTemplate
+from resource_management.core.source import Template
+from resource_management.core.logger import Logger
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons import OSConst
 
+# Local Imports
+
 
 @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
 def yarn(name = None):
@@ -113,6 +127,7 @@ def yarn(name = None):
               recursive_ownership = True,
               )
 
+  #<editor-fold desc="Node Manager Section">
   if name == "nodemanager":
 
     # First start after enabling/disabling security
@@ -156,6 +171,9 @@ def yarn(name = None):
                 recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'},
                 )
 
+    create_hive_llap_work_dir(params)
+  #</editor-fold>
+
   if params.yarn_nodemanager_recovery_dir:
     Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
               owner=params.yarn_user,
@@ -445,3 +463,32 @@ def yarn(name = None):
          owner=params.mapred_user,
          group=params.user_group
     )
+
+
+def create_hive_llap_work_dir(params):
+  """
+  Create the work directory needed by LLAP, which is required on all NodeManagers.
+  This needs to be whenever NodeManagers are restarted, or after Hive Server Interactive and LLAP are added and started
+  via a custom command
+  :param params: Command parameters dictionary.
+  """
+  if params.hive_llap_work_dirs_list is None or params.hive_server_interactive_host is None or not params.security_enabled:
+    Logger.info("Skip creating any Hive LLAP work dir since either it's empty, Hive Interactive is not present, "
+                "or cluster is not kerberized.")
+    return
+
+  # If we already created this dir list because it has the same value as NM Local Dirs, then skip it
+  skip = False
+  if params.nm_local_dirs_list is not None and set(params.nm_local_dirs_list) == set(params.hive_llap_work_dirs_list):
+    skip = True
+
+  if skip:
+    Logger.info(format("Skip creating Hive LLAP Work Dirs since it is equivalent to NM Local Dirs: {nm_local_dirs_list}"))
+  else:
+    Logger.info(format("Hive Server Interactive is present on the cluster, ensure that the Hive LLAP work dirs exist: {hive_llap_work_dirs_list}"))
+    Directory(params.hive_llap_work_dirs_list,
+              owner=params.yarn_user,
+              group=params.user_group,
+              create_parents=True,
+              ignore_failures=False,
+              mode=0775)

ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json

@@ -31,7 +31,8 @@
         {
           "hive-interactive-site": {
             "hadoop.security.authentication": "KERBEROS",
-            "hadoop.security.authorization": "true"
+            "hadoop.security.authorization": "true",
+            "hive.llap.daemon.work.dirs": "/hadoop/llap/local"
           }
         }
       ],

ambari-web/app/controllers/main/service/item.js

@@ -635,6 +635,44 @@ App.MainServiceItemController = Em.Controller.extend(App.SupportClientConfigsDow
     App.showAlertPopup(Em.I18n.t('common.error'), error);
   },
 
+  createYARNDirectories: function(event) {
+    var context = Em.I18n.t('services.service.actions.run.createYARNDirectories');
+    var command = "CREATE_YARN_DIRECTORIES";
+    var component = "NODEMANAGER";
+    var controller = this;
+    var hosts = App.Service.find('YARN').get('hostComponents').filterProperty('componentName', component).mapProperty('hostName');
+    return App.showConfirmationPopup(function() {
+      App.ajax.send({
+        name: 'service.item.executeCustomCommand',
+        sender: controller,
+        data: {
+          command: command,
+          context: context,
+          hosts: hosts.join(','),
+          serviceName: "YARN",
+          componentName: component
+        },
+        success: 'createYARNDirectoriesSuccessCallback',
+        error: 'createYARNDirectoriesErrorCallback'
+      });
+    }, Em.I18n.t('services.service.actions.run.createYARNDirectories.confirmation'), null, Em.I18n.t('popup.confirmation.commonHeader'), Em.I18n.t('ok'), false);
+  },
+  createYARNDirectoriesSuccessCallback : function(data, ajaxOptions, params) {
+    if (data.Requests.id) {
+      App.router.get('backgroundOperationsController').showPopup();
+    }
+  },
+  createYARNDirectoriesErrorCallback : function(data) {
+    var error = Em.I18n.t('services.service.actions.run.executeCustomCommand.error');
+    if (data && data.responseText) {
+      try {
+        var json = $.parseJSON(data.responseText);
+        error += json.message;
+      } catch (err) {}
+    }
+    App.showAlertPopup(Em.I18n.t('common.error'), error);
+  },
+
   /**
    * On click handler for rebalance Hdfs command from items menu
    */

ambari-web/app/messages.js

@@ -1752,9 +1752,12 @@ Em.I18n.translations = {
   'services.service.actions.run.stopLdapKnox.context':'Stop Demo LDAP',
   'services.service.actions.run.startStopLdapKnox.error': 'Error during remote command: ',
 
-  // LLAP Custom Command
+  // Hive Server Interactive custom command to restart LLAP
   'services.service.actions.run.restartLLAP':'Restart LLAP',
-
+  // Node Manager custom command to refresh YARN Apps on Slider
+  'services.service.actions.run.createYARNDirectories':'Create YARN Directories',
+  'services.service.actions.run.createYARNDirectories.confirmation':'Please confirm. This will create necessary folders on the NodeManagers required for running YARN Apps on Slider, such as LLAP. This does not need to restart NodeManagers.',
+  
   'services.service.actions.run.immediateStopHawqService.context':'Stop HAWQ Service (Immediate Mode)',
   'services.service.actions.run.immediateStopHawqService.label':'Stop HAWQ Service (Immediate Mode)',
   'services.service.actions.run.immediateStopHawqSegment.label':'Stop (Immediate Mode)',

ambari-web/app/models/host_component.js

@@ -370,6 +370,13 @@ App.HostComponentActionMap = {
         label: Em.I18n.t('services.service.actions.run.restartLLAP') + ' ∞',
         cssClass: 'icon-refresh'
       },
+      CREATE_YARN_DIRECTORIES: {
+        action: 'createYARNDirectories',
+        customCommand: 'CREATE_YARN_DIRECTORIES',
+        context: Em.I18n.t('services.service.actions.run.createYARNDirectories'),
+        label: Em.I18n.t('services.service.actions.run.createYARNDirectories'),
+        cssClass: 'icon-refresh'
+      },
       REBALANCEHDFS: {
         action: 'rebalanceHdfsNodes',
         customCommand: 'REBALANCEHDFS',

ambari-web/app/views/main/service/item.js

@@ -40,7 +40,8 @@ App.MainServiceItemView = Em.View.extend({
     'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE', 'RUN_HAWQ_CHECK', 'HAWQ_CLEAR_CACHE', 'REMOVE_HAWQ_STANDBY', 'RESYNC_HAWQ_STANDBY'],
     'HAWQSEGMENT': ['IMMEDIATE_STOP_HAWQ_SEGMENT'],
     'HAWQSTANDBY': ['ACTIVATE_HAWQ_STANDBY'],
-    'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"]
+    'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"],
+    'NODEMANAGER' : ["CREATE_YARN_DIRECTORIES"]
   },
 
    addActionMap: function() {
@@ -141,6 +142,16 @@ App.MainServiceItemView = Em.View.extend({
       }
       if (this.get('serviceName') === 'YARN') {
         options.push(actionMap.REFRESHQUEUES);
+        var nodeManagerComponent = App.StackServiceComponent.find().findProperty('componentName', 'NODEMANAGER');
+        var isNodeManagerPresent = allSlaves.contains('NODEMANAGER');
+        if (nodeManagerComponent && isNodeManagerPresent) {
+          var nodeManagerCustomCommands = nodeManagerComponent.get('customCommands');
+          nodeManagerCustomCommands.forEach(function (command) {
+            if (actionMap[command]) {
+              options.push(actionMap[command]);
+            }
+          });
+        }
       }
       options.push(actionMap.RESTART_ALL);
       allSlaves.concat(allMasters).filter(function (_component) {

ambari-web/test/views/main/service/item_test.js

@@ -494,6 +494,8 @@ describe('App.MainServiceItemView', function () {
             return Em.Object.create({ customCommands: ["DECOMMISSION", "REBALANCEHDFS"] });
           case 'RESOURCEMANAGER':
             return Em.Object.create({ customCommands: ["DECOMMISSION", "REFRESHQUEUES"] });
+          case 'NODEMANAGER':
+            return Em.Object.create({ customCommands: ["REFRESHQUEUES", "CREATE_YARN_DIRECTORIES"] });
           case 'HBASE_MASTER':
             return Em.Object.create({ customCommands: ["DECOMMISSION"] });
           case 'KNOX_GATEWAY':