AMBARI-19879. Updating yarn-env and hadoop-env templates with ZK secure options on stack upgrade (Attila Magyar via magyari_sandor)

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py

@@ -39,8 +39,6 @@ from resource_management.libraries.functions.stack_features import check_stack_f
 from resource_management.libraries.script import Script
 from resource_management.core.resources.zkmigrator import ZkMigrator
 
-
-
 class ZkfcSlave(Script):
   def get_component_name(self):
     import params
@@ -62,6 +60,7 @@ class ZkfcSlave(Script):
     import params
     env.set_params(params)
     hdfs("zkfc_slave")
+    utils.set_up_zkfc_security(params)
     pass
 
 @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml

@@ -375,6 +375,11 @@
             <regex-replace  key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/>
             <regex-replace  key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/>
           </definition>
+
+          <definition xsi:type="configure" id="hadoop_env_zkfc_security_opts" summary="Adding HDFS ZKFC Security ACLs">
+            <type>hadoop-env</type>
+            <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS=&quot;{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
+          </definition>
         </changes>
       </component>
     </service>
@@ -504,6 +509,10 @@
             <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/>
             <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/>
           </definition>
+          <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs">
+            <type>yarn-env</type>
+            <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS=&quot;{{rm_security_opts}} $YARN_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
+          </definition>
         </changes>
       </component>
     </service>

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml

@@ -286,6 +286,12 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="HDFS" component="NAMENODE" title="Adding HDFS ZKFC Security ACLs">
+        <task xsi:type="configure" id="hadoop_env_zkfc_security_opts">
+          <summary>Adding HDFS ZKFC Security ACLs</summary>
+        </task>
+      </execute-stage>
+
       <!-- YARN -->
       <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig">
@@ -293,6 +299,12 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security ACLs">
+        <task xsi:type="configure" id="yarn_env_security_opts">
+          <summary>Adding YARN Security ACLs</summary>
+        </task>
+      </execute-stage>
+
       <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for Yarn Resourcemanager">
         <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/>
       </execute-stage>

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml

@@ -680,6 +680,7 @@
           <task xsi:type="configure" id="hdp_2_4_0_0_namenode_ha_adjustments"/>
           <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" />
           <task xsi:type="configure" id="hdfs_log4j_parameterize" />
+          <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" />
         </pre-upgrade>
 
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
@@ -760,6 +761,7 @@
           </task>
           <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db" />
           <task xsi:type="configure" id="yarn_log4j_parameterize" />
+          <task xsi:type="configure" id="yarn_env_security_opts" />
         </pre-upgrade>
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
 

ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml

@@ -264,6 +264,10 @@
             <regex-replace  key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/>
             <regex-replace  key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/>
           </definition>
+          <definition xsi:type="configure" id="hadoop_env_zkfc_security_opts" summary="Adding HDFS ZKFC Security ACLs">
+            <type>hadoop-env</type>
+            <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS=&quot;{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
+          </definition>
         </changes>
       </component>
     </service>
@@ -289,6 +293,10 @@
             <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/>
             <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/>
           </definition>
+          <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs">
+            <type>yarn-env</type>
+            <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS=&quot;{{rm_security_opts}} $YARN_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
+          </definition>
         </changes>
       </component>
 

ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml

@@ -281,6 +281,13 @@
         </task>
       </execute-stage>
 
+      <!--Yarn-->
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security ACLs">
+        <task xsi:type="configure" id="yarn_env_security_opts">
+          <summary>Adding YARN Security ACLs</summary>
+        </task>
+      </execute-stage>
+
       <!-- YARN -->
       <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle">
         <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
@@ -400,6 +407,13 @@
         </task>
       </execute-stage>
 
+      <!--HDFS-->
+      <execute-stage service="HDFS" component="NAMENODE" title="Adding HDFS ZKFC Security ACLs">
+        <task xsi:type="configure" id="hadoop_env_zkfc_security_opts">
+          <summary>Adding HDFS ZKFC Security ACLs</summary>
+        </task>
+      </execute-stage>
+
       <!-- SQOOP -->
       <execute-stage service="SQOOP" component="SQOOP" title="Apply config changes for Sqoop to remove Atlas Configs">
         <!-- Remove Atlas configs that were incorrectly added to sqoop-site instead of Atlas' application.properties. -->

ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml

@@ -685,6 +685,7 @@
           <task xsi:type="configure" id="hdp_2_5_0_0_namenode_ha_adjustments"/>
           <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" />
           <task xsi:type="configure" id="hdfs_log4j_parameterize" />
+          <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" />
         </pre-upgrade>
 
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
@@ -765,6 +766,7 @@
           </task>
           <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db" />
           <task xsi:type="configure" id="yarn_log4j_parameterize" />
+          <task xsi:type="configure" id="yarn_env_security_opts" />
         </pre-upgrade>
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
 

ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml

@@ -128,6 +128,10 @@
           <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/>
           <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/>
         </definition>
+        <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs">
+          <type>yarn-env</type>
+          <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS=&quot;{{rm_security_opts}} $YARN_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
+        </definition>
       </changes>
     </component>
   </service>
@@ -146,6 +150,10 @@
           <regex-replace  key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/>
           <regex-replace  key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/>
         </definition>
+        <definition xsi:type="configure" id="hadoop_env_zkfc_security_opts" summary="Adding HDFS ZKFC Security ACLs">
+          <type>hadoop-env</type>
+          <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS=&quot;{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
+        </definition>
       </changes>
     </component>
   </service>

ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml

@@ -291,6 +291,13 @@
         </task>
       </execute-stage>
 
+      <!--Yarn-->
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security ACLs">
+        <task xsi:type="configure" id="yarn_env_security_opts">
+          <summary>Adding YARN Security ACLs</summary>
+        </task>
+      </execute-stage>
+
       <!--TEZ-->
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath">
@@ -317,6 +324,12 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="HDFS" component="NAMENODE" title="Adding HDFS ZKFC Security ACLs">
+        <task xsi:type="configure" id="hadoop_env_zkfc_security_opts">
+          <summary>Adding HDFS ZKFC Security ACLs</summary>
+        </task>
+      </execute-stage>
+
       <!--HBASE-->
       <execute-stage service="HBASE" component="HBASE_MASTER" title="Parameterizing HBase Log4J Properties">
         <task xsi:type="configure" id="hbase_log4j_parameterize">

ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml

@@ -589,6 +589,7 @@
       <component name="NAMENODE">
         <pre-upgrade>
           <task xsi:type="configure" id="hdfs_log4j_parameterize" />
+          <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" />
         </pre-upgrade>
         <pre-downgrade />
         <upgrade>
@@ -667,6 +668,7 @@
       <component name="RESOURCEMANAGER">
         <pre-upgrade>
           <task xsi:type="configure" id="yarn_log4j_parameterize" />
+          <task xsi:type="configure" id="yarn_env_security_opts" />
         </pre-upgrade>
         <pre-downgrade />
         <upgrade>