HADOOP-477. Extend contrib/streaming to scan the PATH environment variable when resolving executable program names. Contributed by Dhruba Borthakur.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@464685 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -1,6 +1,13 @@
 Hadoop Change Log
 
 
+Trunk (unreleased changes)
+
+ 1. HADOOP-477.  Extend contrib/streaming to scan the PATH environment
+    variables when resolving executable program names.
+    (Dhruba Borthakur via cutting) 
+
+
 Release 0.7.1 - 2006-10-11
 
  1. HADOOP-593.  Fix a NullPointerException in the JobTracker.

src/contrib/streaming/src/java/org/apache/hadoop/streaming/PathFinder.java

@@ -0,0 +1,146 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.streaming;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Maps a relative pathname to an absolute pathname using the
+ * PATH enviroment.
+ * @author Dhruba Borthakur
+ */
+public class PathFinder
+{
+    String pathenv;        // a string of pathnames
+    String pathSep;        // the path seperator
+    String fileSep;        // the file seperator in a directory
+
+    /**
+     * Construct a PathFinder object using the path from
+     * java.class.path
+     */
+    public PathFinder()
+    {
+        pathenv = System.getProperty("java.class.path");
+        pathSep = System.getProperty("path.separator");
+        fileSep = System.getProperty("file.separator");
+    }
+
+    /**
+     * Construct a PathFinder object using the path from
+     * the specified system environment variable.
+     */
+    public PathFinder(String envpath)
+    {
+        pathenv = System.getenv(envpath);
+        pathSep = System.getProperty("path.separator");
+        fileSep = System.getProperty("file.separator");
+    }
+
+    /**
+     * Appends the specified component to the path list
+     */
+    public void prependPathComponent(String str)
+    {
+        pathenv = str + pathSep + pathenv;
+    }
+
+    /**
+     * Returns the full path name of this file if it is listed in the
+     * path
+     */
+    public File getAbsolutePath(String filename)
+    {
+        if (pathenv == null || pathSep == null  || fileSep == null)
+        {
+            return null;
+        }
+        int     val = -1;
+        String    classvalue = pathenv + pathSep;
+
+        while (((val = classvalue.indexOf(pathSep)) >= 0) &&
+               classvalue.length() > 0) {
+            //
+            // Extract each entry from the pathenv
+            //
+            String entry = classvalue.substring(0, val).trim();
+            File f = new File(entry);
+
+            try {
+                if (f.isDirectory()) {
+                    //
+                    // this entry in the pathenv is a directory.
+                    // see if the required file is in this directory
+                    //
+                    f = new File(entry + fileSep + filename);
+                }
+                //
+                // see if the filename matches and  we can read it
+                //
+                if (f.isFile() && f.canRead()) {
+                    return f;
+                }
+            } catch (Exception exp){ }
+            classvalue = classvalue.substring(val+1).trim();
+        }
+        return null;
+    }
+
+    /**
+     * prints all environment variables for this process
+     */
+    private static void printEnvVariables() {
+        System.out.println("Environment Variables: ");
+        Map<String,String> map = System.getenv();
+        Set<String> keys = map.keySet();
+        Iterator iter = keys.iterator();
+        while(iter.hasNext()) {
+            String thiskey = (String)(iter.next()); 
+            String value = map.get(thiskey);
+            System.out.println(thiskey + " = " + value);
+        }
+    }
+
+    /**
+     * prints all system properties for this process
+     */
+    private static void printSystemProperties() {
+        System.out.println("System properties: ");
+        java.util.Properties p = System.getProperties();
+        java.util.Enumeration keys = p.keys();
+        while(keys.hasMoreElements()) {
+            String thiskey = (String)keys.nextElement();
+            String value = p.getProperty(thiskey);
+            System.out.println(thiskey + " = " + value);
+        }
+    }
+
+    public static void main(String args[]) throws IOException {
+
+        if (args.length < 1) {
+            System.out.println("Usage: java PathFinder <filename>");
+            System.exit(1);
+        }
+
+        PathFinder finder = new PathFinder("PATH");
+        File file = finder.getAbsolutePath(args[0]);
+        if (file != null) {
+            System.out.println("Full path name = " + file.getCanonicalPath());
+        }
+    }
+}

src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java

@@ -273,11 +273,23 @@ public abstract class PipeMapRed {
         sideEffectOut_ = getURIOutputStream(sideEffectURI_, allowSocket);
       }
 
+      // 
       // argvSplit[0]:
       // An absolute path should be a preexisting valid path on all TaskTrackers
-      // A  relative path should match in the unjarred Job data
-      // In this case, force an absolute path to make sure exec finds it.
-      argvSplit[0] = new File(argvSplit[0]).getAbsolutePath();
+      // A relative path is converted into an absolute pathname by looking
+      // up the PATH env variable. If it still fails, look it up in the
+      // tasktracker's local working directory
+      //
+      if (!new File(argvSplit[0]).isAbsolute()) {
+          PathFinder finder = new PathFinder("PATH");
+          finder.prependPathComponent(".");
+          File f = finder.getAbsolutePath(argvSplit[0]);
+          if (f != null) {
+              argvSplit[0] = f.getAbsolutePath();
+          }
+          f = null;
+      }
+      System.out.println("XXX2 argvSplit[0] = " + argvSplit[0]);
       logprintln("PipeMapRed exec " + Arrays.asList(argvSplit));
       logprintln("sideEffectURI_=" + sideEffectURI_);