AMBARI-17983. (1). Revert AMBARI-16031 (Create /hadoop/llap/local on each host and disk in Kerberized cluster for LLAP), and (2). Remove the config 'hive.llap.daemon.work.dirs' as HIVE will manage the work directories itself.

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml

@@ -91,16 +91,6 @@
              <!-- Used by decommission and recommission -->
              <masterComponent>RESOURCEMANAGER</masterComponent>
            </bulkCommands>
-          <customCommands>
-            <customCommand>
-              <name>CREATE_YARN_DIRECTORIES</name>
-              <commandScript>
-                <script>scripts/nodemanager.py</script>
-                <scriptType>PYTHON</scriptType>
-                <timeout>600</timeout>
-              </commandScript>
-            </customCommand>
-          </customCommands>
           <logs>
             <log>
               <logId>yarn_nodemanager</logId>

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py

@@ -30,7 +30,7 @@ from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.security_commons import build_expectations, \
   cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
   FILE_TYPE_XML
-from yarn import yarn, create_hive_llap_work_dir
+from yarn import yarn
 from service import service
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyImpl
@@ -89,16 +89,6 @@ class NodemanagerDefault(Nodemanager):
     env.set_params(status_params)
     check_process_status(status_params.nodemanager_pid_file)
 
-  def create_yarn_directories(self, env):
-    """
-    Custom command to Create Directories, e.g., needed by YARN Apps on Slider
-    """
-    Logger.info("Custom Command to Create Directories")
-    import params
-    env.set_params(params)
-    create_hive_llap_work_dir(params)
-
-
   def security_status(self, env):
     import status_params
     env.set_params(status_params)

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py

@@ -19,7 +19,6 @@ Ambari Agent
 
 """
 import os
-import re
 
 from resource_management.libraries.script.script import Script
 from resource_management.libraries.resources.hdfs_resource import HdfsResource
@@ -204,39 +203,12 @@ if hostname and nm_address and nm_address.startswith("0.0.0.0:"):
 nm_local_dirs = default("/configurations/yarn-site/yarn.nodemanager.local-dirs", "")
 nm_log_dirs = default("/configurations/yarn-site/yarn.nodemanager.log-dirs", "")
 
+nm_local_dirs_list = nm_local_dirs.split(',')
+nm_log_dirs_list = nm_log_dirs.split(',')
+
 nm_log_dir_to_mount_file = "/var/lib/ambari-agent/data/yarn/yarn_log_dir_mount.hist"
 nm_local_dir_to_mount_file = "/var/lib/ambari-agent/data/yarn/yarn_local_dir_mount.hist"
 
-# If Hive Interactive Server is present, then default value will be ${yarn.nodemanager.local-dirs}
-# During kerberization, kerberos.json changes the value to "/hadoop/llap/local".
-# This is a comma separated list of directories.
-hive_llap_work_dirs = default("/configurations/hive-interactive-site/hive.llap.daemon.work.dirs", "")
-
-# It can also be a dynamic variable, such as "${yarn.nodemanager.local-dirs}"
-p = re.compile(r"\$\{(.*?)\}")
-m = p.search(hive_llap_work_dirs)
-if m and len(m.groups()) == 1:
-  hive_llap_work_dirs = default("/configurations/yarn-site/" + m.group(1), "")
-
-nm_local_dirs_list = nm_local_dirs.split(",")
-nm_log_dirs_list = nm_log_dirs.split(",")
-hive_llap_work_dirs_list = hive_llap_work_dirs.split(",")
-
-
-def cleanse_list(l):
-  if l is None:
-    return None
-  l = [e.strip() for e in l if e.strip() != ""]
-  return l if len(l) > 0 else None
-
-nm_local_dirs_list = cleanse_list(nm_local_dirs_list)
-nm_log_dirs_list = cleanse_list(nm_log_dirs_list)
-hive_llap_work_dirs_list = cleanse_list(hive_llap_work_dirs_list)
-
-# Needed for LLAP to determine if should create Hive LLAP Work Dirs.
-hive_server_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', [])
-hive_server_interactive_host = hive_server_interactive_hosts[0] if len(hive_server_interactive_hosts) > 0 else None
-
 distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar"
 hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"
 

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py

@@ -202,8 +202,6 @@ def yarn(name=None, config_dir=None):
            mode=0644,
            content=nm_local_dir_to_mount_file_content
       )
-
-    create_hive_llap_work_dir(params)
   #</editor-fold>
 
   if params.yarn_nodemanager_recovery_dir:
@@ -498,33 +496,4 @@ def yarn(name=None, config_dir=None):
     File(os.path.join(config_dir, 'ssl-server.xml.example'),
          owner=params.mapred_user,
          group=params.user_group
-    )
-
-
-def create_hive_llap_work_dir(params):
-  """
-  Create the work directory needed by LLAP, which is required on all NodeManagers.
-  This needs to be whenever NodeManagers are restarted, or after Hive Server Interactive and LLAP are added and started
-  via a custom command
-  :param params: Command parameters dictionary.
-  """
-  if params.hive_llap_work_dirs_list is None or params.hive_server_interactive_host is None or not params.security_enabled:
-    Logger.info("Skip creating any Hive LLAP work dir since either it's empty, Hive Interactive is not present, "
-                "or cluster is not kerberized.")
-    return
-
-  # If we already created this dir list because it has the same value as NM Local Dirs, then skip it
-  skip = False
-  if params.nm_local_dirs_list is not None and set(params.nm_local_dirs_list) == set(params.hive_llap_work_dirs_list):
-    skip = True
-
-  if skip:
-    Logger.info(format("Skip creating Hive LLAP Work Dirs since it is equivalent to NM Local Dirs: {nm_local_dirs_list}"))
-  else:
-    Logger.info(format("Hive Server Interactive is present on the cluster, ensure that the Hive LLAP work dirs exist: {hive_llap_work_dirs_list}"))
-    Directory(params.hive_llap_work_dirs_list,
-              owner=params.yarn_user,
-              group=params.user_group,
-              create_parents=True,
-              ignore_failures=False,
-              mode=0775)
+    )

ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml

@@ -606,16 +606,6 @@ limitations under the License.
     </description>
     <on-ambari-upgrade add="true"/>
   </property>
-  <property>
-    <name>hive.llap.daemon.work.dirs</name>
-    <value>${yarn.nodemanager.local-dirs}</value>
-    <description>
-      Working directories for the daemon. Needs to be set for a secure cluster, since LLAP may
-      not have access to the default YARN working directories. yarn.nodemanager.local-dirs is
-      used if this is not set
-    </description>
-    <on-ambari-upgrade add="true"/>
-  </property>
   <property>
     <name>hive.llap.zk.sm.connectionString</name>
     <value>localhost:2181</value>

ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json

@@ -28,11 +28,6 @@
             "hadoop.proxyuser.HTTP.hosts": "${clusterHostInfo/webhcat_server_host}"
           }
         },
-        {
-          "hive-interactive-site": {
-            "hive.llap.daemon.work.dirs": "/hadoop/llap/local"
-          }
-        },
         {
           "ranger-hive-audit": {
             "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule",

ambari-web/app/controllers/main/service/item.js

@@ -693,44 +693,6 @@ App.MainServiceItemController = Em.Controller.extend(App.SupportClientConfigsDow
     App.showAlertPopup(Em.I18n.t('common.error'), error);
   },
 
-  createYARNDirectories: function(event) {
-    var context = Em.I18n.t('services.service.actions.run.createYARNDirectories');
-    var command = "CREATE_YARN_DIRECTORIES";
-    var component = "NODEMANAGER";
-    var controller = this;
-    var hosts = App.Service.find('YARN').get('hostComponents').filterProperty('componentName', component).mapProperty('hostName');
-    return App.showConfirmationPopup(function() {
-      App.ajax.send({
-        name: 'service.item.executeCustomCommand',
-        sender: controller,
-        data: {
-          command: command,
-          context: context,
-          hosts: hosts.join(','),
-          serviceName: "YARN",
-          componentName: component
-        },
-        success: 'createYARNDirectoriesSuccessCallback',
-        error: 'createYARNDirectoriesErrorCallback'
-      });
-    }, Em.I18n.t('services.service.actions.run.createYARNDirectories.confirmation'), null, Em.I18n.t('popup.confirmation.commonHeader'), Em.I18n.t('ok'), false);
-  },
-  createYARNDirectoriesSuccessCallback : function(data, ajaxOptions, params) {
-    if (data.Requests.id) {
-      App.router.get('backgroundOperationsController').showPopup();
-    }
-  },
-  createYARNDirectoriesErrorCallback : function(data) {
-    var error = Em.I18n.t('services.service.actions.run.executeCustomCommand.error');
-    if (data && data.responseText) {
-      try {
-        var json = $.parseJSON(data.responseText);
-        error += json.message;
-      } catch (err) {}
-    }
-    App.showAlertPopup(Em.I18n.t('common.error'), error);
-  },
-
   /**
    * On click handler for rebalance Hdfs command from items menu
    */

ambari-web/app/messages.js

@@ -1816,9 +1816,6 @@ Em.I18n.translations = {
 
   // Hive Server Interactive custom command to restart LLAP
   'services.service.actions.run.restartLLAP':'Restart LLAP',
-  // Node Manager custom command to refresh YARN Apps on Slider
-  'services.service.actions.run.createYARNDirectories':'Create YARN Directories',
-  'services.service.actions.run.createYARNDirectories.confirmation':'Please confirm. This will create necessary folders on the NodeManagers required for running YARN Apps on Slider, such as LLAP. This does not need to restart NodeManagers.',
   'services.service.actions.run.immediateStopHawqService.context':'Stop HAWQ Service (Immediate Mode)',
   'services.service.actions.run.immediateStopHawqService.label':'Stop HAWQ Service (Immediate Mode)',
   'services.service.actions.run.immediateStopHawqSegment.label':'Stop (Immediate Mode)',

ambari-web/app/models/host_component.js

@@ -372,13 +372,6 @@ App.HostComponentActionMap = {
         label: Em.I18n.t('services.service.actions.run.restartLLAP') + ' ∞',
         cssClass: 'icon-refresh'
       },
-      CREATE_YARN_DIRECTORIES: {
-        action: 'createYARNDirectories',
-        customCommand: 'CREATE_YARN_DIRECTORIES',
-        context: Em.I18n.t('services.service.actions.run.createYARNDirectories'),
-        label: Em.I18n.t('services.service.actions.run.createYARNDirectories'),
-        cssClass: 'icon-refresh'
-      },
       REBALANCEHDFS: {
         action: 'rebalanceHdfsNodes',
         customCommand: 'REBALANCEHDFS',

ambari-web/app/views/main/service/item.js

@@ -40,8 +40,7 @@ App.MainServiceItemView = Em.View.extend({
     'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE', 'RUN_HAWQ_CHECK', 'HAWQ_CLEAR_CACHE', 'REMOVE_HAWQ_STANDBY', 'RESYNC_HAWQ_STANDBY'],
     'HAWQSEGMENT': ['IMMEDIATE_STOP_HAWQ_SEGMENT'],
     'HAWQSTANDBY': ['ACTIVATE_HAWQ_STANDBY'],
-    'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"],
-    'NODEMANAGER' : ["CREATE_YARN_DIRECTORIES"]
+    'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"]
   },
 
    addActionMap: function() {
@@ -142,16 +141,6 @@ App.MainServiceItemView = Em.View.extend({
       }
       if (this.get('serviceName') === 'YARN') {
         options.push(actionMap.REFRESHQUEUES);
-        var nodeManagerComponent = App.StackServiceComponent.find().findProperty('componentName', 'NODEMANAGER');
-        var isNodeManagerPresent = allSlaves.contains('NODEMANAGER');
-        if (nodeManagerComponent && isNodeManagerPresent) {
-          var nodeManagerCustomCommands = nodeManagerComponent.get('customCommands');
-          nodeManagerCustomCommands.forEach(function (command) {
-            if (actionMap[command]) {
-              options.push(actionMap[command]);
-            }
-          });
-        }
       }
       options.push(actionMap.RESTART_ALL);
       allSlaves.concat(allMasters).filter(function (_component) {

ambari-web/test/views/main/service/item_test.js

@@ -494,8 +494,6 @@ describe('App.MainServiceItemView', function () {
             return Em.Object.create({ customCommands: ["DECOMMISSION", "REBALANCEHDFS"] });
           case 'RESOURCEMANAGER':
             return Em.Object.create({ customCommands: ["DECOMMISSION", "REFRESHQUEUES"] });
-          case 'NODEMANAGER':
-            return Em.Object.create({ customCommands: ["REFRESHQUEUES", "CREATE_YARN_DIRECTORIES"] });
           case 'HBASE_MASTER':
             return Em.Object.create({ customCommands: ["DECOMMISSION"] });
           case 'KNOX_GATEWAY':