AMBARI-14764. MapReduce, Oozie service check failures after upgrade finished (dlysnichenko)

Conflicts:
	ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml

ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java

@@ -258,40 +258,37 @@ public class HeartbeatMonitor implements Runnable {
         continue;
       }
 
-      if (clusterConfig != null) {
-        // cluster config for 'global'
-        Map<String, String> props = new HashMap<String, String>(clusterConfig.getProperties());
+      // cluster config for 'global'
+      Map<String, String> props = new HashMap<>(clusterConfig.getProperties());
 
-        Map<String, Map<String, String>> configTags = new HashMap<String,
-                Map<String, String>>();
+      Map<String, Map<String, String>> configTags = new HashMap<>();
 
-        for (Map.Entry<String, Map<String, String>> entry : allConfigTags.entrySet()) {
-          if (entry.getKey().equals(clusterConfig.getType())) {
-            configTags.put(clusterConfig.getType(), entry.getValue());
-          }
+      for (Map.Entry<String, Map<String, String>> entry : allConfigTags.entrySet()) {
+        if (entry.getKey().equals(clusterConfig.getType())) {
+          configTags.put(clusterConfig.getType(), entry.getValue());
         }
+      }
 
-        Map<String, Map<String, String>> properties = configHelper
-                .getEffectiveConfigProperties(cluster, configTags);
+      Map<String, Map<String, String>> properties = configHelper
+              .getEffectiveConfigProperties(cluster, configTags);
 
-        if (!properties.isEmpty()) {
-          for (Map<String, String> propertyMap : properties.values()) {
-            props.putAll(propertyMap);
-          }
+      if (!properties.isEmpty()) {
+        for (Map<String, String> propertyMap : properties.values()) {
+          props.putAll(propertyMap);
         }
+      }
 
-        configurations.put(clusterConfig.getType(), props);
+      configurations.put(clusterConfig.getType(), props);
 
-        Map<String, Map<String, String>> attrs = new TreeMap<String, Map<String, String>>();
-        configHelper.cloneAttributesMap(clusterConfig.getPropertiesAttributes(), attrs);
+      Map<String, Map<String, String>> attrs = new TreeMap<>();
+      configHelper.cloneAttributesMap(clusterConfig.getPropertiesAttributes(), attrs);
 
-        Map<String, Map<String, Map<String, String>>> attributes = configHelper
-            .getEffectiveConfigAttributes(cluster, configTags);
-        for (Map<String, Map<String, String>> attributesMap : attributes.values()) {
-          configHelper.cloneAttributesMap(attributesMap, attrs);
-        }
-        configurationAttributes.put(clusterConfig.getType(), attrs);
+      Map<String, Map<String, Map<String, String>>> attributes = configHelper
+          .getEffectiveConfigAttributes(cluster, configTags);
+      for (Map<String, Map<String, String>> attributesMap : attributes.values()) {
+        configHelper.cloneAttributesMap(attributesMap, attrs);
       }
+      configurationAttributes.put(clusterConfig.getType(), attrs);
     }
 
     StatusCommand statusCmd = new StatusCommand();

ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java

@@ -216,6 +216,14 @@ public enum CheckDescription {
           "The following config types will have values overwritten: %s");
       }}),
 
+  HARDCODED_STACK_VERSION_PROPERTIES_CHECK(PrereqCheckType.CLUSTER,
+    "Found hardcoded hdp stack version in property value.",
+    new HashMap<String, String>() {{
+      put(AbstractCheckDescriptor.DEFAULT,
+        "Some properties seem to contain hardcoded hdp version string \"%s\"." +
+          " That is a potential problem when doing stack update.");
+      }}),
+
   SERVICES_RANGER_PASSWORD_VERIFY(PrereqCheckType.SERVICE,
       "Verify Ambari and Ranger Password Synchronization",
       new HashMap<String, String>() {{

ambari-server/src/main/java/org/apache/ambari/server/checks/HardcodedStackVersionPropertiesCheck.java

@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.checks;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.PrereqCheckRequest;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.ConfigMergeHelper;
+import org.apache.ambari.server.state.ConfigMergeHelper.ThreeWayValue;
+import org.apache.ambari.server.state.DesiredConfig;
+import org.apache.ambari.server.state.stack.PrereqCheckStatus;
+import org.apache.ambari.server.state.stack.PrerequisiteCheck;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Checks for properties that contain hardcoded CURRENT hdp version string.
+ * Presence of such properties usually means that some paths are hardcoded to
+ * point to concrete version of HDP, instead of pointing to current symlink.
+ * That is a potential problem when doing stack update.
+ */
+@Singleton
+@UpgradeCheck(order = 98.0f, required = true)
+public class HardcodedStackVersionPropertiesCheck extends AbstractCheckDescriptor {
+
+  @Inject
+  private Provider<Clusters> m_clusters;
+
+  @Inject
+  private Provider<ConfigHelper> m_config_helper_provider;
+
+  public HardcodedStackVersionPropertiesCheck() {
+    super(CheckDescription.HARDCODED_STACK_VERSION_PROPERTIES_CHECK);
+  }
+
+  @Override
+  public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request)
+      throws AmbariException {
+
+    String stackName = request.getTargetStackId().getStackName();
+    RepositoryVersionEntity rve = repositoryVersionDaoProvider.get().
+      findByStackNameAndVersion(stackName, request.getRepositoryVersion());
+
+    Cluster cluster = m_clusters.get().getCluster(request.getClusterName());
+
+    String currentHdpVersion = cluster.getCurrentClusterVersion().getRepositoryVersion().getVersion();
+
+    Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs();
+    Set<String> failures = new HashSet<>();
+
+    Pattern searchPattern = getHardcodeSearchPattern(currentHdpVersion);
+    for (Entry<String, DesiredConfig> configEntry : desiredConfigs.entrySet()) {
+      String configType = configEntry.getKey();
+      DesiredConfig desiredConfig = configEntry.getValue();
+      final Config config = cluster.getConfig(configType, desiredConfig.getTag());
+
+      Map<String, String> properties = config.getProperties();
+      for (Entry<String, String> property : properties.entrySet()) {
+        if (stringContainsVersionHardcode(property.getValue(), searchPattern)) {
+          failures.add(String.format(" %s/%s",
+            configType, property.getKey()));
+        }
+      }
+
+      if (failures.size() > 0) {
+        prerequisiteCheck.setStatus(PrereqCheckStatus.WARNING);
+        String failReason = getFailReason(prerequisiteCheck, request);
+
+        prerequisiteCheck.setFailReason(String.format(failReason, currentHdpVersion));
+        prerequisiteCheck.setFailedOn(new LinkedHashSet<>(failures));
+
+      } else {
+        prerequisiteCheck.setStatus(PrereqCheckStatus.PASS);
+      }
+    }
+  }
+
+  /**
+   * Returns pattern that looks for hdp version hardcoded occurences, except
+   * those that start with "-Dhdp.version=" string
+   * @param hdpVersion hdp version to search for
+   * @return compiled pattern
+   */
+  public static Pattern getHardcodeSearchPattern(String hdpVersion) {
+    // Only things like -Dhdp.version=2.3.4.0-1234 are allowed at hardcode
+    return Pattern.compile("(?<!-Dhdp\\.version=)" + hdpVersion.replace(".", "\\."));
+  }
+
+  /**
+   * Looks for search pattern in string
+   * @param string string to look into
+   * @param searchPattern compiled regex
+   * @return true if string contains pattern
+   */
+  public static boolean stringContainsVersionHardcode(String string, Pattern searchPattern) {
+    Matcher matcher = searchPattern.matcher(string);
+    return matcher.find();
+  }
+
+}

ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixLzoCodecPath.java

@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.upgrades;
+
+import com.google.inject.Inject;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * During stack upgrade, update lzo codec path in mapreduce.application.classpath and
+ * at tez.cluster.additional.classpath.prefix to look like
+ * /usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar
+ */
+public class FixLzoCodecPath extends AbstractServerAction {
+
+  /**
+   * Lists config types and properties that may contain lzo codec path
+   */
+  private static final HashMap<String, String []> TARGET_PROPERTIES = new HashMap<String, String []>() {{
+    put("mapred-site", new String [] {"mapreduce.application.classpath"});
+    put("tez-site", new String [] {"tez.cluster.additional.classpath.prefix"});
+  }};
+
+  @Inject
+  private Clusters clusters;
+
+  @Override
+  public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+    throws AmbariException, InterruptedException {
+    String clusterName = getExecutionCommand().getClusterName();
+    Cluster cluster = clusters.getCluster(clusterName);
+
+    ArrayList<String> modifiedProperties = new ArrayList<>();
+
+    for (Map.Entry<String, String[]> target : TARGET_PROPERTIES.entrySet()) {
+      Config config = cluster.getDesiredConfigByType(target.getKey());
+      if (config == null) {
+        continue; // Config not found, skip it
+      }
+      Map<String, String> properties = config.getProperties();
+      for (String propertyName : target.getValue()) {
+        String oldContent = properties.get(propertyName);
+        String newContent = fixLzoJarPath(oldContent);
+
+        if (! newContent.equals(oldContent)) {
+          properties.put(propertyName, newContent);
+          modifiedProperties.add(propertyName);
+        }
+      }
+      config.setProperties(properties);
+      config.persist(false);
+    }
+    if (modifiedProperties.isEmpty()) {
+      return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+        "No properties require lzo codec path fixes", "");
+    } else {
+      return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+        String.format("Fixed lzo codec path value at property [%s] to " +
+          "use ${hdp.version} instead of hardcoded HDP version.",
+          StringUtils.join(modifiedProperties, ", ")), "");
+    }
+
+  }
+
+  public static String fixLzoJarPath(String oldPropertyValue) {
+    // Makes sure that LZO codec path uses ${hdp.version} instead of hardcoded hdp version,
+    // so it replaces variations of /usr/hdp/2.3.4.0-3485/hadoop/lib/hadoop-lzo-0.6.0.2.3.4.0-3485.jar
+    // with /usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar
+    return oldPropertyValue.replaceAll(
+      "(/usr/hdp/)[^\\\\:]+(/hadoop/lib/hadoop-lzo-(\\d\\.)+)(\\$\\{hdp.version\\}|(\\d\\.){3}\\d-\\d+)(\\.jar)",
+      "$1\\$\\{hdp.version\\}$2\\$\\{hdp.version\\}$6");
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/OozieConfigCalculation.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.upgrades;
+
+import com.google.inject.Inject;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Changes oozie-env during upgrade (adds -Dhdp.version to $HADOOP_OPTS variable)
+ */
+public class OozieConfigCalculation extends AbstractServerAction {
+  private static final String TARGET_CONFIG_TYPE = "oozie-env";
+  private static final String CONTENT_PROPERTY_NAME = "content";
+
+  @Inject
+  private Clusters clusters;
+
+  @Override
+  public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+    throws AmbariException, InterruptedException {
+    String clusterName = getExecutionCommand().getClusterName();
+    Cluster cluster = clusters.getCluster(clusterName);
+    Config config = cluster.getDesiredConfigByType(TARGET_CONFIG_TYPE);
+
+    if (config == null) {
+      return  createCommandReport(0, HostRoleStatus.FAILED,"{}",
+                                   String.format("Source type %s not found", TARGET_CONFIG_TYPE), "");
+    }
+
+    Map<String, String> properties = config.getProperties();
+    String oldContent = properties.get(CONTENT_PROPERTY_NAME);
+
+    String newContent = processPropertyValue(oldContent);
+
+    if (newContent.equals(oldContent)) {
+      return  createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+        "-Dhdp.version option has been already added to $HADOOP_OPTS variable", "");
+    } else {
+      properties.put(CONTENT_PROPERTY_NAME, newContent);
+    }
+
+    config.setProperties(properties);
+    config.persist(false);
+
+    return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+                  String.format("Added -Dhdp.version to $HADOOP_OPTS variable at %s", TARGET_CONFIG_TYPE), "");
+
+  }
+
+  public static String processPropertyValue(String oldContent) {
+    // For regex simplicity, will not work with multiline export definitions that are
+    // split on few strings using \ character
+    // False negative should be a less bit of trouble (just duplicate exports/option definitions)
+    // than false positive (broken Oozie after upgrade)
+    Pattern regex = Pattern.compile("^export HADOOP_OPTS=.*-Dhdp.version=.*$", Pattern.MULTILINE);
+    Matcher regexMatcher = regex.matcher(oldContent);
+    if (regexMatcher.find()) {
+      return oldContent;
+    } else {
+      StringBuilder newContent = new StringBuilder(oldContent);
+      newContent.append("\n").append(
+        "export HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\" "
+      );
+      return newContent.toString();
+    }
+  }
+}

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-env.xml

@@ -178,6 +178,9 @@ export OOZIE_ADMIN_PORT={{oozie_server_admin_port}}
 # export OOZIE_BASE_URL="http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie"
 export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64
 
+# Set Hadoop-related properties
+export HADOOP_OPTS="-Dhdp.version=${HDP_VERSION} ${HADOOP_OPTS}"
+
    </value>
    <value-attributes>
      <type>content</type>

ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/configuration/oozie-env.xml

@@ -99,6 +99,9 @@ export OOZIE_CLIENT_OPTS="${OOZIE_CLIENT_OPTS} -Doozie.connection.retry.count=5
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}"
 export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}"
 {% endif %}
+
+# Set Hadoop-related properties
+export HADOOP_OPTS="-Dhdp.version=${HDP_VERSION} ${HADOOP_OPTS}"
     </value>
     <value-attributes>
       <type>content</type>

ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml

@@ -252,6 +252,10 @@
         <task xsi:type="configure" id="hdp_2_3_0_0_mapreduce2_adjust_history_server"/>
       </execute-stage>
 
+      <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
       <execute-stage service="YARN" component="APP_TIMELINE_SERVER" title="Apply config changes for AppTimelineServer">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_ats_enable_recovery"/>
       </execute-stage>
@@ -306,6 +310,10 @@
         <task xsi:type="configure" id="hdp_2_3_0_0_tez_keep_ats_v1"/>
       </execute-stage>
 
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
       <!--HIVE-->
       <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="server_action" summary="Calculating ZooKeeper Quorum Properties for Hive" class="org.apache.ambari.server.serveraction.upgrades.HiveZKQuorumConfigAction" />
@@ -340,6 +348,10 @@
         <task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/>
       </execute-stage>
 
+      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
+        <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+      </execute-stage>
+
       <!--FALCON-->
       <execute-stage service="FALCON" component="FALCON_SERVER" title="Apply config changes for Falcon Server">
         <task xsi:type="configure" id="hdp_2_2_0_0_falcon_application_services"/>

ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/configuration/oozie-env.xml

@@ -101,6 +101,9 @@ export CATALINA_OPTS="${CATALINA_OPTS} -Xmx2048m -XX:MaxPermSize=256m "
 export CATALINA_OPTS="${CATALINA_OPTS} -Xmx2048m"
 {% endif %}
 
+# Set Hadoop-related properties
+export HADOOP_OPTS="-Dhdp.version=${HDP_VERSION} ${HADOOP_OPTS}"
+
     </value>
     <value-attributes>
       <type>content</type>

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml

@@ -236,9 +236,21 @@
       <direction>UPGRADE</direction>   <!--  prevent config changes on downgrade -->
       <skippable>true</skippable>  <!-- May fix configuration problems manually -->
 
+      <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
         <task xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_lib_uris_property"/>
       </execute-stage>
+
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
+      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
+        <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+      </execute-stage>
     </group>
 
     <!-- Now, restart all of the services. -->

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml

@@ -340,6 +340,10 @@
         <task xsi:type="configure" id="hdp_2_3_0_0_mapreduce2_adjust_history_server"/>
       </execute-stage>
 
+      <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
       <execute-stage service="YARN" component="APP_TIMELINE_SERVER" title="Apply config changes for AppTimelineServer">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_ats_enable_recovery"/>
       </execute-stage>
@@ -429,6 +433,10 @@
         <task xsi:type="configure" id="hdp_2_3_0_0_tez_keep_ats_v1"/>
       </execute-stage>
 
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
       <!--HIVE-->
       <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_replace_auth_manager"/>
@@ -471,6 +479,9 @@
         <task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/>
       </execute-stage>
 
+      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
+        <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+      </execute-stage>
 
       <!--KNOX-->
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for Knox Gateway">
@@ -1205,4 +1216,4 @@
       </component>
     </service>
   </processing>
-</upgrade>
+</upgrade>

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml

@@ -371,6 +371,10 @@
         <task xsi:type="configure" id="hdp_2_3_0_0_mapreduce2_adjust_history_server"/>
       </execute-stage>
 
+      <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
       <execute-stage service="YARN" component="APP_TIMELINE_SERVER" title="Apply config changes for AppTimelineServer">
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_ats_enable_recovery"/>
       </execute-stage>
@@ -463,6 +467,10 @@
         <task xsi:type="configure" id="hdp_2_3_0_0_tez_keep_ats_v1"/>
       </execute-stage>
 
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
       <!--HIVE-->
       <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_replace_auth_manager"/>
@@ -509,6 +517,10 @@
         <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
       </execute-stage>
 
+      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
+        <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+      </execute-stage>
+
       <!--KNOX-->
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for Knox Gateway">
         <task xsi:type="configure" id="hdp_2_3_0_0_knox_configure_ranger_policy"/>

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml

@@ -442,6 +442,10 @@
       </component>
 
       <component name="MAPREDUCE2_CLIENT">
+        <pre-upgrade>
+          <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+        </pre-upgrade>
+
         <upgrade>
           <task xsi:type="restart-task"/>
         </upgrade>
@@ -497,6 +501,8 @@
       <component name="TEZ_CLIENT">
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_lib_uris_property" />
+
+          <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task"/>
@@ -598,6 +604,8 @@
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
+          
+          <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
 
           <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml

@@ -530,6 +530,10 @@
       </component>
 
       <component name="MAPREDUCE2_CLIENT">
+        <pre-upgrade>
+          <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+        </pre-upgrade>
+
         <upgrade>
           <task xsi:type="restart-task"/>
         </upgrade>
@@ -632,6 +636,7 @@
           <task xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties"/>
           <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_tez_lib_uris_property"/>
           <task xsi:type="configure" id="hdp_2_3_0_0_tez_keep_ats_v1"/>
+          <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task"/>
@@ -744,6 +749,8 @@
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/>
 
+          <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+
           <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml

@@ -536,6 +536,10 @@
       </component>
 
       <component name="MAPREDUCE2_CLIENT">
+        <pre-upgrade>
+          <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+        </pre-upgrade>
+
         <upgrade>
           <task xsi:type="restart-task"/>
         </upgrade>
@@ -643,6 +647,8 @@
           <task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/>
 
           <task xsi:type="configure" id="hdp_2_3_0_0_tez_keep_ats_v1"/>
+
+          <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task"/>
@@ -756,6 +762,8 @@
           <task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/>
           <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
 
+          <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+
           <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml

@@ -267,6 +267,16 @@
       <direction>UPGRADE</direction>   <!--  prevent config changes on downgrade -->
       <skippable>true</skippable>  <!-- May fix configuration problems manually -->
 
+      <!--YARN-->
+      <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
+      <!--TEZ-->
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
       <!-- HBASE -->
       <!-- These HBASE configs changed in HDP 2.3.4.0, but Ambari can't distinguish HDP 2.3.2.0 vs HDP 2.3.4.0, so easier to always do them. -->
       <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBASE Master">
@@ -276,6 +286,11 @@
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
         <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_tez_lib_uris_property"/>
       </execute-stage>
+
+      <!--OOZIE-->
+      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
+        <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+      </execute-stage>
     </group>
 
     <!-- Now, restart all of the services. -->

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml

@@ -278,17 +278,26 @@
         <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
       </execute-stage>
 
+      <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
       <!-- HBASE -->
       <!-- These HBASE configs changed in HDP 2.3.4.0, but Ambari can't distinguish HDP 2.3.2.0 vs HDP 2.3.4.0, so easier to always do them. -->
       <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBASE Master">
         <task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/>
       </execute-stage>
 
-      <!-- HIVE, TEZ, -->
+      <!-- TEZ -->
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
         <task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/>
       </execute-stage>
 
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
+      <!-- HIVE -->
       <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for WebHCat Server">
         <task xsi:type="configure" id="hdp_2_4_0_0_webhcat_server_update_configuration_paths"/>
       </execute-stage>
@@ -297,7 +306,11 @@
         <task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/>
       </execute-stage>
 
-      <!-- OOZIE -->
+      <!--OOZIE-->
+      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
+        <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+      </execute-stage>
+
       <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
         <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
       </execute-stage>
@@ -308,7 +321,6 @@
         <task xsi:type="configure" id ="hdp_2_4_0_0_kafka_broker_listeners"/>
       </execute-stage>
 
-
     </group>
 
     <!-- Now, restart all of the services. -->

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml

@@ -525,6 +525,10 @@
       </component>
 
       <component name="MAPREDUCE2_CLIENT">
+        <pre-upgrade>
+          <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+        </pre-upgrade>
+
         <upgrade>
           <task xsi:type="restart-task"/>
         </upgrade>
@@ -592,6 +596,8 @@
       <component name="TEZ_CLIENT">
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_tez_lib_uris_property"/>
+
+          <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task"/>
@@ -699,6 +705,8 @@
             <function>stop</function>
           </task>
 
+          <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+
           <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml

@@ -480,6 +480,10 @@
       </component>
 
       <component name="MAPREDUCE2_CLIENT">
+        <pre-upgrade>
+          <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+        </pre-upgrade>
+
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>
@@ -556,6 +560,8 @@
       <component name="TEZ_CLIENT">
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/>
+
+          <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
@@ -678,6 +684,8 @@
       <component name="OOZIE_SERVER">
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
+
+          <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
         
           <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>

ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml

@@ -268,9 +268,24 @@
       <direction>UPGRADE</direction>   <!--  prevent config changes on downgrade -->
       <skippable>true</skippable>  <!-- May fix configuration problems manually -->
 
+      <!--YARN-->
+      <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
+      <!--TEZ-->
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez">
+        <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+      </execute-stage>
+
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
         <task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/>
       </execute-stage>
+
+      <!--OOZIE-->
+      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
+        <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+      </execute-stage>
     </group>
 
     <!-- Now, restart all of the services. -->

ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml

@@ -523,6 +523,10 @@
       </component>
 
       <component name="MAPREDUCE2_CLIENT">
+        <pre-upgrade>
+          <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
+        </pre-upgrade>
+
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>
@@ -585,6 +589,8 @@
       <component name="TEZ_CLIENT">
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/>
+
+          <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
@@ -692,6 +698,8 @@
             <function>stop</function>
           </task>
 
+          <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+
           <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>

ambari-server/src/test/java/org/apache/ambari/server/checks/HardcodedStackVersionPropertiesCheckTest.java

@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.checks;
+
+import org.junit.Test;
+
+import java.util.regex.Pattern;
+
+import static org.junit.Assert.*;
+
+public class HardcodedStackVersionPropertiesCheckTest {
+
+  private static final String currentVersion = "2.3.4.0-1234";
+
+  @Test
+  public void testGetHardcodeSearchPattern() throws Exception {
+    Pattern p = HardcodedStackVersionPropertiesCheck.getHardcodeSearchPattern(currentVersion);
+    assertEquals(p.pattern(), "(?<!-Dhdp\\.version=)2\\.3\\.4\\.0-1234");
+  }
+
+  @Test
+  public void testStringContainsVersionHardcode() throws Exception {
+    Pattern pattern = HardcodedStackVersionPropertiesCheck.getHardcodeSearchPattern(currentVersion);
+
+    // Check various cases
+    String content = "";
+    assertFalse(HardcodedStackVersionPropertiesCheck.stringContainsVersionHardcode(content, pattern));
+
+    content = "2.3.4.0-1234";
+    assertTrue(HardcodedStackVersionPropertiesCheck.stringContainsVersionHardcode(content, pattern));
+
+    content = "dfsdfds fdsfds -Dhdp.version=2.3.4.0-1234 sfdfdsfds";
+    assertFalse(HardcodedStackVersionPropertiesCheck.stringContainsVersionHardcode(content, pattern));
+
+    content = "dfsdfds fdsfds -Dhdp.version=2.3.4.0-1234 \n sfdfdsfds 2.3.4.0-1234 \n fdsfds";
+    assertTrue(HardcodedStackVersionPropertiesCheck.stringContainsVersionHardcode(content, pattern));
+
+    content = "hdp.version=2.3.4.0-1234";
+    assertTrue(HardcodedStackVersionPropertiesCheck.stringContainsVersionHardcode(content, pattern));
+
+    content = "kgflkfld fdf\nld;ls;f d hdp.version=2.3.4.0-1234 \n sfdfdsfds \n fdsfds";
+    assertTrue(HardcodedStackVersionPropertiesCheck.stringContainsVersionHardcode(content, pattern));
+  }
+}

ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixLzoCodecPathTest.java

@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests FixLzoCodecPath logic
+ */
+public class FixLzoCodecPathTest {
+
+  @Test
+  public void testFixLzoJarPath() throws Exception {
+    // Test replacements in a big list of paths (typical mapreduce.application.classpath value)
+    String oldContent = "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:" +
+      "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:" +
+      "$PWD/mr-framework/hadoop/share/hadoop/common/*:" +
+      "$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:" +
+      "$PWD/mr-framework/hadoop/share/hadoop/yarn/*:/usr/hdp/2.3.4.0-3485/hadoop/lib/hadoop-lzo-0.6.0.2.3.4.0-3485.jar:" +
+      "$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:" +
+      "$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:" +
+      "/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure";
+    String newContent = FixLzoCodecPath.fixLzoJarPath(oldContent);
+    assertEquals("$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:" +
+      "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:" +
+      "$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:" +
+      "$PWD/mr-framework/hadoop/share/hadoop/yarn/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:" +
+      "$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:" +
+      "$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:" +
+      "/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure", newContent);
+    
+    // Test replacements in typical tez.cluster.additional.classpath.prefix value
+    oldContent = "/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure";
+    newContent = FixLzoCodecPath.fixLzoJarPath(oldContent);
+    assertEquals("/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure", newContent);
+
+    oldContent = "/usr/hdp/2.3.4.0-3485/hadoop/lib/hadoop-lzo-0.6.0.2.3.4.0-345.jar:/etc/hadoop/conf/secure";
+    newContent = FixLzoCodecPath.fixLzoJarPath(oldContent);
+    assertEquals("/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure", newContent);
+
+    oldContent = "/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.2.3.4.0-345.jar:/etc/hadoop/conf/secure";
+    newContent = FixLzoCodecPath.fixLzoJarPath(oldContent);
+    assertEquals("/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure", newContent);
+
+    oldContent = "/usr/hdp/2.3.4.0-3485/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure";
+    newContent = FixLzoCodecPath.fixLzoJarPath(oldContent);
+    assertEquals("/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure", newContent);
+
+    // Other corner cases tests
+    oldContent = "/usr/hdp/2.3.4.0-3485/hadoop/lib/hadoop-lzo-0.6.0.2.3.4.0-1.jar";
+    newContent = FixLzoCodecPath.fixLzoJarPath(oldContent);
+    assertEquals("/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar", newContent);
+
+    oldContent = "/etc/hadoop/conf/secure";
+    newContent = FixLzoCodecPath.fixLzoJarPath(oldContent);
+    assertEquals("/etc/hadoop/conf/secure", newContent);
+  }
+
+}

ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/OozieConfigCalculationTest.java

@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import org.apache.commons.lang.StringUtils;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests OozieConfigCalculation logic
+ */
+public class OozieConfigCalculationTest {
+
+  /**
+   * Checks that -Dhdp.version is added to $HADOOP_OPTS variable at oozie-env
+   * content.
+   * Also checks that it is not added multiple times during upgrades
+   * @throws Exception
+   */
+  @Test
+  public void testOozieEnvWithMissingParam() throws Exception {
+    // Test case when old content does not contain $HADOOP_OPTS variable at all
+    String oldContent = "#!/bin/bash\n" +
+      "\n" +
+      "if [ -d \"/usr/lib/bigtop-tomcat\" ]; then\n" +
+      "  export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf}\n" +
+      "  export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}}\n" +
+      "  export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie}\n" +
+      "  export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat\n" +
+      "fi\n" +
+      "\n" +
+      "# export OOZIE_BASE_URL=\"http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie\"\n" +
+      "export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64";
+    String newContent = OozieConfigCalculation.processPropertyValue(oldContent);
+    assertTrue(newContent.endsWith("export HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\" "));
+    // Test case when old content contains proper $HADOOP_OPTS variable
+    oldContent = newContent;
+    newContent = OozieConfigCalculation.processPropertyValue(oldContent);
+    assertEquals(newContent, oldContent);
+    assertEquals(1, StringUtils.countMatches(newContent, "-Dhdp.version"));
+    // Test case when old content contains $HADOOP_OPTS variable with some value
+    oldContent = "#!/bin/bash\n" +
+      "\n" +
+      "if [ -d \"/usr/lib/bigtop-tomcat\" ]; then\n" +
+      "  export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf}\n" +
+      "  export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}}\n" +
+      "  export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie}\n" +
+      "  export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat\n" +
+      "  export HADOOP_OPTS=-Dsome.option1 -Dsome.option1 $HADOOP_OPTS\n" +
+      "fi\n" +
+      "\n" +
+      "# export OOZIE_BASE_URL=\"http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie\"\n" +
+      "export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64";
+    newContent = OozieConfigCalculation.processPropertyValue(oldContent);
+    assertEquals("#!/bin/bash\n" +
+      "\n" +
+      "if [ -d \"/usr/lib/bigtop-tomcat\" ]; then\n" +
+      "  export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf}\n" +
+      "  export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}}\n" +
+      "  export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie}\n" +
+      "  export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat\n" +
+      "  export HADOOP_OPTS=-Dsome.option1 -Dsome.option1 $HADOOP_OPTS\n" +
+      "fi\n" +
+      "\n" +
+      "# export OOZIE_BASE_URL=\"http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie\"\n" +
+      "export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64\n" +
+      "export HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\" ", newContent);
+  }
+
+}