Sfoglia il codice sorgente

YARN-8817. [Submarine] In cases when user doesn't ask HDFS path while submitting job but framework requires user to set HDFS related environments. Contributed by Wangda Tan.

Sunil G 6 anni fa
parent
commit
d0bc6a27fd

+ 10 - 6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java

@@ -102,6 +102,13 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
     }
   }
 
+  private boolean needHdfs(String content) {
+    if (content != null && content.contains("hdfs://")) {
+      return true;
+    }
+    return false;
+  }
+
   private void addHdfsClassPathIfNeeded(RunJobParameters parameters,
       PrintWriter fw, Component comp) throws IOException {
     // Find envs to use HDFS
@@ -121,14 +128,11 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
 
     boolean lackingEnvs = false;
 
-    if ((parameters.getInputPath() != null && parameters.getInputPath()
-        .contains("hdfs://")) || (parameters.getCheckpointPath() != null
-        && parameters.getCheckpointPath().contains("hdfs://")) || (
-        parameters.getSavedModelPath() != null && parameters.getSavedModelPath()
-            .contains("hdfs://")) || hadoopEnv) {
+    if (needHdfs(parameters.getInputPath()) || needHdfs(
+        parameters.getPSLaunchCmd()) || needHdfs(
+        parameters.getWorkerLaunchCmd()) || hadoopEnv) {
       // HDFS is asked either in input or output, set LD_LIBRARY_PATH
       // and classpath
-
       if (hdfsHome != null) {
         // Unset HADOOP_HOME/HADOOP_YARN_HOME to make sure host machine's envs
         // won't pollute docker's env.