HADOOP-19456. Upgrade kafka to 3.9.0 to fix CVE-2024-31141. (#7416) Contributed by Palakur Eshwitha Sai.

Signed-off-by: Shilun Fan <slfan1989@apache.org>

LICENSE-binary

@@ -317,7 +317,7 @@ org.apache.htrace:htrace-core:3.1.0-incubating
 org.apache.htrace:htrace-core4:4.1.0-incubating
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.kafka:kafka-clients:3.4.0
+org.apache.kafka:kafka-clients:3.9.0
 org.apache.kerby:kerb-admin:2.0.3
 org.apache.kerby:kerb-client:2.0.3
 org.apache.kerby:kerb-common:2.0.3
@@ -377,7 +377,7 @@ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/com
 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h
 
-com.github.luben:zstd-jni:1.5.2-1
+com.github.luben:zstd-jni:1.5.6-4
 dnsjava:dnsjava:3.6.1
 org.codehaus.woodstox:stax2-api:4.2.1
 

hadoop-project/pom.xml

@@ -50,7 +50,7 @@
     <!-- Version number for xerces used by JDiff -->
     <xerces.jdiff.version>2.12.2</xerces.jdiff.version>
 
-    <kafka.version>3.4.0</kafka.version>
+    <kafka.version>3.9.0</kafka.version>
 
     <commons-daemon.version>1.0.13</commons-daemon.version>
 

hadoop-tools/hadoop-kafka/src/main/java/org/apache/hadoop/metrics2/sink/KafkaSink.java

@@ -111,6 +111,8 @@ public class KafkaSink implements MetricsSink, Closeable {
       LOG.warn("Error getting Hostname, going to continue");
     }
 
+    System.setProperty("org.apache.kafka.automatic.config.providers", "none");
+
     try {
       // Create the producer object.
       producer = new KafkaProducer<Integer, byte[]>(props);