Selaa lähdekoodia

AMBARI-16857. Support yarn aux service classpath isolation feature in spark (Bikas Saha via smohanty)

Sumit Mohanty 9 vuotta sitten
vanhempi
commit
6411a0db63

+ 4 - 0
ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml

@@ -291,6 +291,10 @@
         <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/>
       </execute-stage>
 
+      <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle">
+        <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+      </execute-stage>
+
       <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client">
         <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
       </execute-stage>

+ 3 - 0
ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml

@@ -590,6 +590,9 @@
       </component>
 
       <component name="NODEMANAGER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+        </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>

+ 5 - 0
ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml

@@ -277,6 +277,11 @@
         <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/>
       </execute-stage>
 
+      <!-- YARN -->
+      <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle">
+        <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+      </execute-stage>
+
       <!--TEZ-->
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez">
         <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>

+ 3 - 0
ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml

@@ -584,6 +584,9 @@
       </component>
 
       <component name="NODEMANAGER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+        </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>

+ 43 - 0
ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml

@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<configuration supports_final="true">
+    <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>mapreduce_shuffle,spark_shuffle,spark2_shuffle</value>
+        <description>Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and can
+          not start with numbers</description>
+    </property>
+
+    <property>
+        <name>yarn.nodemanager.aux-services.spark2_shuffle.class</name>
+        <value>org.apache.spark.network.yarn.YarnShuffleService</value>
+        <description>The auxiliary service class to use for Spark 2</description>
+    </property>
+
+    <property>
+      <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
+      <value>/usr/hdp/${hdp.version}/spark/aux/*</value>
+    </property>
+
+    <property>
+      <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
+      <value>/usr/hdp/${hdp.version}/spark2/aux/*</value>
+    </property>
+</configuration>

+ 11 - 0
ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml

@@ -50,6 +50,17 @@
         </changes>
       </component>
     </service>
+    <service name="YARN">
+      <component name="NODEMANAGER">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle">
+            <type>yarn-site</type>
+            <set key="yarn.nodemanager.aux-services" value="mapreduce_shuffle,spark_shuffle,spark2_shuffle"/>
+            <!-- Ideally we need to append spark2_shuffle to the existing value -->
+          </definition>
+        </changes>
+      </component>
+    </service>
   </services>
 
 </upgrade-config-changes>