AMBARI-12276. Ozzie running with pig fail to renew JHS delegation token when RU because JHS recovery is not enabled (dlysnichenko)

ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java

@@ -152,6 +152,19 @@ public enum CheckDescription {
           "YARN should have state preserving restart enabled for the Timeline server. The yarn-site.xml property yarn.timeline-service.recovery.enabled should be set to true.");
       }}),
 
+  SERVICES_MR2_JOBHISTORY_ST(PrereqCheckType.SERVICE,
+      "MapReduce2 JobHistory recovery should be enabled",
+      new HashMap<String, String>() {{
+        put(MapReduce2JobHistoryStatePreservingCheck.MAPREDUCE2_JOBHISTORY_RECOVERY_ENABLE_KEY,
+          "MapReduce2 should have recovery enabled for the JobHistory server. The mapred-site.xml property mapreduce.jobhistory.recovery.enable should be set to true.");
+        put(MapReduce2JobHistoryStatePreservingCheck.MAPREDUCE2_JOBHISTORY_RECOVERY_STORE_KEY,
+          "MapReduce2 should have recovery enabled for the JobHistory server. The mapred-site.xml property mapreduce.jobhistory.recovery.store.class should be set to org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService.");
+        put(MapReduce2JobHistoryStatePreservingCheck.MAPREDUCE2_JOBHISTORY_RECOVERY_STORE_LEVELDB_PATH_KEY,
+          "MapReduce2 should have recovery enabled for the JobHistory server. The mapred-site.xml property mapreduce.jobhistory.recovery.store.leveldb.path should be set. Please note that \"mapreduce.jobhistory.recovery.store.leveldb.path\" should be set equal to \"yarn.timeline-service.leveldb-state-store.path\" for consistency.");
+        put(MapReduce2JobHistoryStatePreservingCheck.YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH_KEY,
+          "The mapred-site.xml property mapreduce.jobhistory.recovery.store.leveldb.path should be set. Please note that \"mapreduce.jobhistory.recovery.store.leveldb.path\" should be set equal to \"yarn.timeline-service.leveldb-state-store.path\" for consistency.");
+      }}),
+
   SERVICES_HIVE_DYNAMIC_SERVICE_DISCOVERY(PrereqCheckType.SERVICE,
       "Hive Dynamic Service Discovery",
       new HashMap<String, String>() {{

ambari-server/src/main/java/org/apache/ambari/server/checks/MapReduce2JobHistoryStatePreservingCheck.java

@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.checks;
+
+import com.google.inject.Singleton;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.PrereqCheckRequest;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.stack.PrereqCheckStatus;
+import org.apache.ambari.server.state.stack.PrerequisiteCheck;
+import org.apache.ambari.server.utils.VersionUtils;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The {@link MapReduce2JobHistoryStatePreservingCheck}
+ * is used to check that the MR2 History server has state preserving mode enabled.
+ */
+@Singleton
+@UpgradeCheck(group = UpgradeCheckGroup.CONFIGURATION_WARNING, order = 1.0f)
+public class MapReduce2JobHistoryStatePreservingCheck extends AbstractCheckDescriptor {
+
+  final static String MAPREDUCE2_JOBHISTORY_RECOVERY_ENABLE_KEY =
+    "mapreduce.jobhistory.recovery.enable";
+  final static String MAPREDUCE2_JOBHISTORY_RECOVERY_STORE_KEY =
+    "mapreduce.jobhistory.recovery.store.class";
+  final static String MAPREDUCE2_JOBHISTORY_RECOVERY_STORE_LEVELDB_PATH_KEY =
+    "mapreduce.jobhistory.recovery.store.leveldb.path";
+  final static String YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH_KEY =
+    "yarn.timeline-service.leveldb-state-store.path";
+
+  /**
+   * Due to the introduction of MapReduce2 JobHistory state recovery only from certain
+   * stack-versions onwards, this check is not applicable to earlier versions
+   * of the stack.
+   *
+   * This enumeration lists the minimum stack-versions for which this check is applicable.
+   * If a stack is not specified in this enumeration, this check will be applicable.
+   */
+  private enum MinimumApplicableStackVersion {
+    HDP_STACK("HDP", "2.3.0.0");
+
+    private String stackName;
+    private String stackVersion;
+
+    private MinimumApplicableStackVersion(String stackName, String stackVersion) {
+      this.stackName = stackName;
+      this.stackVersion = stackVersion;
+    }
+
+    public String getStackName() {
+      return stackName;
+    }
+
+    public String getStackVersion() {
+      return stackVersion;
+    }
+  }
+
+  /**
+   * Constructor.
+   */
+  public MapReduce2JobHistoryStatePreservingCheck() {
+    super(CheckDescription.SERVICES_MR2_JOBHISTORY_ST);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public boolean isApplicable(PrereqCheckRequest request) throws AmbariException {
+    if (!super.isApplicable(request)) {
+      return false;
+    }
+
+    final Cluster cluster = clustersProvider.get().getCluster(request.getClusterName());
+    Map<String, Service> services = cluster.getServices();
+    if (!services.containsKey("MAPREDUCE2")) {
+      return false;
+    }
+
+    // Applicable only if stack not defined in MinimumApplicableStackVersion, or
+    // version equals or exceeds the enumerated version.
+    for (MinimumApplicableStackVersion minimumStackVersion : MinimumApplicableStackVersion.values()) {
+      String stackName = cluster.getCurrentStackVersion().getStackName();
+      if (minimumStackVersion.getStackName().equals(stackName)){
+        String targetVersion = request.getTargetStackId().getStackVersion();
+        return VersionUtils.compareVersions(targetVersion, minimumStackVersion.getStackVersion()) >= 0;
+      }
+    }
+
+    return true;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request) throws AmbariException {
+    List<String> errorMessages = new ArrayList<String>();
+    PrereqCheckStatus checkStatus = PrereqCheckStatus.FAIL;
+
+    String enabled =
+      getProperty(request, "mapred-site", MAPREDUCE2_JOBHISTORY_RECOVERY_ENABLE_KEY);
+    String storeClass =
+      getProperty(request, "mapred-site", MAPREDUCE2_JOBHISTORY_RECOVERY_STORE_KEY);
+    String storeLevelDbPath =
+      getProperty(request, "mapred-site", MAPREDUCE2_JOBHISTORY_RECOVERY_STORE_LEVELDB_PATH_KEY);
+    String yarnLevelDbPath =
+      getProperty(request, "yarn-site", YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH_KEY);
+
+    if (null == enabled || !Boolean.parseBoolean(enabled)) {
+      errorMessages.add(getFailReason(MAPREDUCE2_JOBHISTORY_RECOVERY_ENABLE_KEY, prerequisiteCheck, request));
+    }
+
+    if (StringUtils.isBlank(storeClass)) {
+      errorMessages.add(getFailReason(MAPREDUCE2_JOBHISTORY_RECOVERY_STORE_KEY, prerequisiteCheck,
+        request));
+    }
+
+    if (StringUtils.isBlank(storeLevelDbPath)) {
+      errorMessages.add(getFailReason(MAPREDUCE2_JOBHISTORY_RECOVERY_STORE_LEVELDB_PATH_KEY, prerequisiteCheck,
+        request));
+
+    } else if (!storeLevelDbPath.equals(yarnLevelDbPath)) {
+      checkStatus = PrereqCheckStatus.WARNING;
+      errorMessages.add(getFailReason(YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH_KEY, prerequisiteCheck,
+        request));
+    }
+
+    if (!errorMessages.isEmpty()) {
+      prerequisiteCheck.setFailReason(StringUtils.join(errorMessages, "\n"));
+      prerequisiteCheck.getFailedOn().add("MAPREDUCE2");
+      prerequisiteCheck.setStatus(checkStatus);
+    }
+  }
+}

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py

@@ -214,6 +214,7 @@ yarn_nm_app_log_dir =  config['configurations']['yarn-site']['yarn.nodemanager.r
 mapreduce_jobhistory_intermediate_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.intermediate-done-dir']
 mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.done-dir']
 jobhistory_heapsize = default("/configurations/mapred-env/jobhistory_heapsize", "900")
+jhs_leveldb_state_store_dir = default('/configurations/mapred-site/mapreduce.jobhistory.recovery.store.leveldb.path', "/hadoop/yarn/timeline")
 
 # Tez-related properties
 tez_user = config['configurations']['tez-env']['tez_user']

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py

@@ -90,6 +90,12 @@ def yarn(name = None):
                          mode=0777
     )
     params.HdfsResource(None, action="execute")
+    Directory(params.jhs_leveldb_state_store_dir,
+              owner=params.mapred_user,
+              group=params.user_group,
+              recursive=True,
+              cd_access="a",
+              )
 
   if name == "nodemanager":
     Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','),

ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration-mapred/mapred-site.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration supports_final="true" xmlns:xi="http://www.w3.org/2001/XInclude">
+
+  <property>
+    <name>mapreduce.jobhistory.recovery.enable</name>
+    <value>true</value>
+    <description>Enable the history server to store server state and recover
+      server state upon startup.  If enabled then
+      mapreduce.jobhistory.recovery.store.class must be specified.
+    </description>
+  </property>
+
+  <property>
+    <name>mapreduce.jobhistory.recovery.store.class</name>
+    <value>org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService</value>
+    <description>The HistoryServerStateStoreService class to store history server
+      state for recovery.
+    </description>
+  </property>
+
+  <property>
+    <name>mapreduce.jobhistory.recovery.store.leveldb.path</name>
+    <value>/hadoop/yarn/timeline</value>
+    <description>The URI where history server state will be stored if HistoryServerLeveldbSystemStateStoreService
+      is configured as the recovery storage class.
+    </description>
+  </property>
+
+</configuration>

ambari-server/src/test/java/org/apache/ambari/server/checks/MapReduce2JobHistoryStatePreservingCheckTest.java

@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.checks;
+
+import com.google.inject.Provider;
+import org.apache.ambari.server.controller.PrereqCheckRequest;
+import org.apache.ambari.server.orm.entities.ClusterVersionEntity;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.DesiredConfig;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.stack.PrereqCheckStatus;
+import org.apache.ambari.server.state.stack.PrerequisiteCheck;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Tests for {@link org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheckTest}
+ */
+public class MapReduce2JobHistoryStatePreservingCheckTest {
+  private final Clusters m_clusters = Mockito.mock(Clusters.class);
+
+  private final MapReduce2JobHistoryStatePreservingCheck m_check = new MapReduce2JobHistoryStatePreservingCheck();
+
+  /**
+   *
+   */
+  @Before
+  public void setup() {
+    m_check.clustersProvider = new Provider<Clusters>() {
+
+      @Override
+      public Clusters get() {
+        return m_clusters;
+      }
+    };
+  }
+
+  /**
+   * @throws Exception
+   */
+  @Test
+  public void testIsApplicable() throws Exception {
+    final Cluster cluster = Mockito.mock(Cluster.class);
+    Mockito.when(cluster.getClusterId()).thenReturn(1L);
+    Mockito.when(m_clusters.getCluster("cluster")).thenReturn(cluster);
+    Mockito.when(cluster.getCurrentStackVersion()).thenReturn(new StackId("HDP-2.2"));
+
+    Map<String, Service> services = new HashMap<String, Service>();
+    Mockito.when(cluster.getServices()).thenReturn(services);
+
+    ClusterVersionEntity clusterVersionEntity = Mockito.mock(ClusterVersionEntity.class);
+    Mockito.when(cluster.getCurrentClusterVersion()).thenReturn(clusterVersionEntity);
+
+    PrereqCheckRequest request = new PrereqCheckRequest("cluster");
+    request.setTargetStackId(new StackId("HDP", "2.3.0.0"));
+
+    // MAPREDUCE2 not installed
+    Assert.assertFalse(m_check.isApplicable(request));
+
+    // MAPREDUCE2 installed
+    services.put("MAPREDUCE2", Mockito.mock(Service.class));
+    Assert.assertTrue(m_check.isApplicable(request));
+
+    request.setTargetStackId(new StackId("HDP", "2.2.0.1"));
+    Assert.assertFalse(m_check.isApplicable(request));
+  }
+
+  @Test
+  public void testPerform() throws Exception {
+    final Cluster cluster = Mockito.mock(Cluster.class);
+    Mockito.when(cluster.getClusterId()).thenReturn(1L);
+    Mockito.when(m_clusters.getCluster("cluster")).thenReturn(cluster);
+
+    final DesiredConfig desiredConfig = Mockito.mock(DesiredConfig.class);
+    Mockito.when(desiredConfig.getTag()).thenReturn("tag");
+    Map<String, DesiredConfig> configMap = new HashMap<String, DesiredConfig>();
+    configMap.put("mapred-site", desiredConfig);
+    configMap.put("yarn-site", desiredConfig);
+
+    Mockito.when(cluster.getDesiredConfigs()).thenReturn(configMap);
+    final Config config = Mockito.mock(Config.class);
+    Mockito.when(cluster.getConfig(Mockito.anyString(), Mockito.anyString())).thenReturn(config);
+    final Map<String, String> properties = new HashMap<String, String>();
+    Mockito.when(config.getProperties()).thenReturn(properties);
+
+    PrerequisiteCheck check = new PrerequisiteCheck(null, null);
+    m_check.perform(check, new PrereqCheckRequest("cluster"));
+    Assert.assertEquals(PrereqCheckStatus.FAIL, check.getStatus());
+
+    properties.put(MapReduce2JobHistoryStatePreservingCheck.MAPREDUCE2_JOBHISTORY_RECOVERY_ENABLE_KEY, "true");
+    properties.put(MapReduce2JobHistoryStatePreservingCheck.MAPREDUCE2_JOBHISTORY_RECOVERY_STORE_KEY, "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService");
+    properties.put(MapReduce2JobHistoryStatePreservingCheck.MAPREDUCE2_JOBHISTORY_RECOVERY_STORE_LEVELDB_PATH_KEY, "/hadoop/yarn/timeline");
+    check = new PrerequisiteCheck(null, null);
+    m_check.perform(check, new PrereqCheckRequest("cluster"));
+    Assert.assertEquals(PrereqCheckStatus.WARNING, check.getStatus());
+    check = new PrerequisiteCheck(null, null);
+    properties.put(MapReduce2JobHistoryStatePreservingCheck.YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH_KEY, "not /hadoop/yarn/timeline");
+    m_check.perform(check, new PrereqCheckRequest("cluster"));
+    Assert.assertEquals(PrereqCheckStatus.WARNING, check.getStatus());
+    check = new PrerequisiteCheck(null, null);
+    properties.put(MapReduce2JobHistoryStatePreservingCheck.YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH_KEY, "/hadoop/yarn/timeline");
+    m_check.perform(check, new PrereqCheckRequest("cluster"));
+    Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus());
+  }
+
+  @SuppressWarnings("serial")
+  @Test
+  public void testIsApplicableMinimumStackVersion() throws Exception {
+    final Cluster cluster = Mockito.mock(Cluster.class);
+    Mockito.when(cluster.getClusterId()).thenReturn(1L);
+    Mockito.when(cluster.getServices()).thenReturn(new HashMap<String, Service>() {
+      {
+        put("MAPREDUCE2", null);
+      }
+    });
+    Mockito.when(cluster.getCurrentStackVersion()).thenReturn(new StackId("MYSTACK-12.2"));
+    ClusterVersionEntity clusterVersionEntity = Mockito.mock(ClusterVersionEntity.class);
+    Mockito.when(cluster.getCurrentClusterVersion()).thenReturn(clusterVersionEntity);
+    RepositoryVersionEntity repositoryVersionEntity = Mockito.mock(RepositoryVersionEntity.class);
+    Mockito.when(clusterVersionEntity.getRepositoryVersion()).thenReturn(repositoryVersionEntity);
+    Mockito.when(m_clusters.getCluster("c1")).thenReturn(cluster);
+    PrereqCheckRequest request = new PrereqCheckRequest("c1");
+
+    Mockito.when(repositoryVersionEntity.getVersion()).thenReturn("2.2.0.1");
+    boolean isApplicable = m_check.isApplicable(request);
+    Assert.assertTrue(isApplicable);
+  }
+}

ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py

@@ -247,6 +247,12 @@ class TestHistoryServer(RMFTestCase):
         action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020',
         hadoop_conf_dir = '/etc/hadoop/conf',
     )
+    self.assertResourceCalled('Directory', '/hadoop/yarn/timeline',
+      owner = 'mapred',
+      group = 'hadoop',
+      recursive = True,
+      cd_access = 'a',
+    )
     self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
       owner = 'yarn',
       group = 'hadoop',
@@ -464,6 +470,12 @@ class TestHistoryServer(RMFTestCase):
         action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020',
         hadoop_conf_dir = '/etc/hadoop/conf',
     )
+    self.assertResourceCalled('Directory', '/hadoop/yarn/timeline',
+      owner = 'mapred',
+      group = 'hadoop',
+      recursive = True,
+      cd_access = 'a',
+    )
     self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
       owner = 'yarn',
       group = 'hadoop',