HADOOP-9872. Improve protoc version handling and detection. (tucu)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1514074 13f79535-47bb-0310-9956-ffa450edef68

BUILDING.txt

@@ -7,7 +7,7 @@ Requirements:
 * JDK 1.6
 * Maven 3.0
 * Findbugs 1.3.9 (if running findbugs)
-* ProtocolBuffer 2.4.1+ (for MapReduce and HDFS)
+* ProtocolBuffer 2.5.0
 * CMake 2.6 or newer (if compiling native code)
 * Internet connection for first build (to fetch all Maven and Hadoop dependencies)
 
@@ -99,6 +99,16 @@ level once; and then work from the submodule. Keep in mind that SNAPSHOTs
 time out after a while, using the Maven '-nsu' will stop Maven from trying
 to update SNAPSHOTs from external repos.
 
+----------------------------------------------------------------------------------
+Protocol Buffer compiler
+
+The version of Protocol Buffer compiler, protoc, must match the version of the
+protobuf JAR.
+
+If you have multiple versions of protoc in your system, you can set in your 
+build shell the HADOOP_PROTOC_PATH environment variable to point to the one you 
+want to use for the Hadoop build. If you don't define this environment variable,
+protoc is looked up in the PATH.
 ----------------------------------------------------------------------------------
 Importing projects to eclipse
 

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -315,6 +315,10 @@ Release 2.1.0-beta - 2013-08-06
     HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs
     (todd)
 
+    HADOOP-9845. Update protobuf to 2.5 from 2.4.x. (tucu)
+
+    HADOOP-9872. Improve protoc version handling and detection. (tucu)
+
   BUG FIXES
 
     HADOOP-9451. Fault single-layer config if node group topology is enabled.

hadoop-common-project/hadoop-common/pom.xml

@@ -309,6 +309,7 @@
             </goals>
             <configuration>
               <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
               <imports>
                 <param>${basedir}/src/main/proto</param>
               </imports>
@@ -338,6 +339,7 @@
             </goals>
             <configuration>
               <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
               <imports>
                 <param>${basedir}/src/test/proto</param>
               </imports>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java

@@ -90,6 +90,10 @@ public class VersionInfo {
       " source checksum " + _getSrcChecksum();
   }
 
+  protected String _getProtocVersion() {
+    return info.getProperty("protocVersion", "Unknown");
+  }
+
   private static VersionInfo COMMON_VERSION_INFO = new VersionInfo("common");
   /**
    * Get the Hadoop version.
@@ -153,12 +157,20 @@ public class VersionInfo {
   public static String getBuildVersion(){
     return COMMON_VERSION_INFO._getBuildVersion();
   }
-  
+
+  /**
+   * Returns the protoc version used for the build.
+   */
+  public static String getProtocVersion(){
+    return COMMON_VERSION_INFO._getProtocVersion();
+  }
+
   public static void main(String[] args) {
     LOG.debug("version: "+ getVersion());
     System.out.println("Hadoop " + getVersion());
     System.out.println("Subversion " + getUrl() + " -r " + getRevision());
     System.out.println("Compiled by " + getUser() + " on " + getDate());
+    System.out.println("Compiled with protoc " + getProtocVersion());
     System.out.println("From source with checksum " + getSrcChecksum());
     System.out.println("This command was run using " + 
         ClassUtil.findContainingJar(VersionInfo.class));

hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties

@@ -23,3 +23,4 @@ user=${user.name}
 date=${version-info.build.time}
 url=${version-info.scm.uri}
 srcChecksum=${version-info.source.md5}
+protocVersion=${protobuf.version}

hadoop-common-project/hadoop-common/src/site/apt/CLIMiniCluster.apt.vm

@@ -42,7 +42,7 @@ Hadoop MapReduce Next Generation - CLI MiniCluster.
 $ mvn clean install -DskipTests
 $ mvn package -Pdist -Dtar -DskipTests -Dmaven.javadoc.skip
 +---+
-  <<NOTE:>> You will need protoc installed of version 2.4.1 or greater.
+  <<NOTE:>> You will need protoc 2.5.0 installed.
 
   The tarball should be available in <<<hadoop-dist/target/>>> directory. 
 

hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm

@@ -32,7 +32,7 @@ $ mvn clean install -DskipTests
 $ cd hadoop-mapreduce-project
 $ mvn clean install assembly:assembly -Pnative
 +---+
-  <<NOTE:>> You will need protoc installed of version 2.4.1 or greater.
+  <<NOTE:>> You will need protoc 2.5.0 installed.
 
   To ignore the native builds in mapreduce you can omit the <<<-Pnative>>> argument
   for maven. The tarball should be available in <<<target/>>> directory. 

hadoop-hdfs-project/hadoop-hdfs/pom.xml

@@ -412,7 +412,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
               <goal>protoc</goal>
             </goals>
             <configuration>
-              <protocVersion>2.5.0</protocVersion>
+              <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
               <imports>
                 <param>${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto</param>
                 <param>${basedir}/src/main/proto</param>
@@ -437,7 +438,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
               <goal>protoc</goal>
             </goals>
             <configuration>
-              <protocVersion>2.5.0</protocVersion>
+              <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
               <imports>
                 <param>${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto</param>
                 <param>${basedir}/src/main/proto</param>
@@ -459,7 +461,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
               <goal>protoc</goal>
             </goals>
             <configuration>
-              <protocVersion>2.5.0</protocVersion>
+              <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
               <imports>
                 <param>${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto</param>
                 <param>${basedir}/src/main/proto</param>
@@ -481,7 +484,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
               <goal>protoc</goal>
             </goals>
             <configuration>
-              <protocVersion>2.5.0</protocVersion>
+              <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
               <imports>
                 <param>${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto</param>
                 <param>${basedir}/src/main/proto</param>

hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml

@@ -103,7 +103,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
               <goal>protoc</goal>
             </goals>
             <configuration>
-              <protocVersion>2.5.0</protocVersion>
+              <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
               <imports>
                 <param>${basedir}/../../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
                 <param>${basedir}/../../../../../hadoop-hdfs-project/hadoop-hdfs/src/main/proto</param>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml

@@ -64,7 +64,8 @@
               <goal>protoc</goal>
             </goals>
             <configuration>
-              <protocVersion>2.5.0</protocVersion>
+              <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
               <imports>
                 <param>${basedir}/../../../hadoop-common-project/hadoop-common/src/main/proto</param>
                 <param>${basedir}/../../../hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto</param>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml

@@ -78,7 +78,8 @@
 				<goal>protoc</goal>
 				</goals>
 				<configuration>
-                                  <protocVersion>2.5.0</protocVersion>
+        <protocVersion>${protobuf.version}</protocVersion>
+        <protocCommand>${protoc.path}</protocCommand>
 				<imports>
 					<param>
 						${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto

hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java

@@ -45,7 +45,7 @@ public class ProtocMojo extends AbstractMojo {
   @Parameter(required=true)
   private FileSet source;
 
-  @Parameter(defaultValue="protoc")
+  @Parameter
   private String protocCommand;
 
   @Parameter(required=true)
@@ -53,21 +53,22 @@ public class ProtocMojo extends AbstractMojo {
 
   public void execute() throws MojoExecutionException {
     try {
+      if (protocCommand == null || protocCommand.trim().isEmpty()) {
+        protocCommand = "protoc";
+      }
       List<String> command = new ArrayList<String>();
       command.add(protocCommand);
       command.add("--version");
       Exec exec = new Exec(this);
       List<String> out = new ArrayList<String>();
-      if (exec.run(command, out) != 0) {
-        getLog().error("protoc, could not get version");
-        for (String s : out) {
-          getLog().error(s);
-        }
+      if (exec.run(command, out) == 127) {
+        getLog().error("protoc, not found at: " + protocCommand);
         throw new MojoExecutionException("protoc failure");        
       } else {
-        if (out.size() == 0) {
+        if (out.isEmpty()) {
+          getLog().error("stdout: " + out);
           throw new MojoExecutionException(
-              "'protoc -version' did not return a version");
+              "'protoc --version' did not return a version");
         } else {
           if (!out.get(0).endsWith(protocVersion)) {
             throw new MojoExecutionException(

hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java

@@ -63,11 +63,10 @@ public class Exec {
         for (String s : stdErr.getOutput()) {
           mojo.getLog().debug(s);
         }
-      } else {
-        stdOut.join();
-        stdErr.join();
-        output.addAll(stdOut.getOutput());
       }
+      stdOut.join();
+      stdErr.join();
+      output.addAll(stdOut.getOutput());
     } catch (Exception ex) {
       mojo.getLog().warn(command + " failed: " + ex.toString());
     }

hadoop-project/pom.xml

@@ -62,6 +62,7 @@
     <!-- ProtocolBuffer version, used to verify the protoc version and -->
     <!-- define the protobuf JAR version                               -->
     <protobuf.version>2.5.0</protobuf.version>
+    <protoc.path>${env.HADOOP_PROTOC_PATH}</protoc.path>
   </properties>
 
   <dependencyManagement>

hadoop-yarn-project/hadoop-yarn/README

@@ -8,15 +8,9 @@ Maven: Maven 3
 
 Setup
 -----
-Install protobuf 2.4.0a or higher (Download from http://code.google.com/p/protobuf/downloads/list)
+Install protobuf 2.5.0 (Download from http://code.google.com/p/protobuf/downloads/list)
  - install the protoc executable (configure, make, make install)
  - install the maven artifact (cd java; mvn install)
-Installing protoc requires gcc 4.1.x or higher.
-If the make step fails with (Valid until a fix is released for protobuf 2.4.0a)
-    ./google/protobuf/descriptor.h:1152: error:
-    `google::protobuf::internal::Mutex*google::protobuf::DescriptorPool::mutex_'
-    is private
-  Replace descriptor.cc with http://protobuf.googlecode.com/svn-history/r380/trunk/src/google/protobuf/descriptor.cc
 
 
 Quick Maven Tips

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml

@@ -45,7 +45,8 @@
               <goal>protoc</goal>
             </goals>
             <configuration>
-              <protocVersion>2.5.0</protocVersion>
+              <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
               <imports>
                 <param>${basedir}/../../../hadoop-common-project/hadoop-common/src/main/proto</param>
                 <param>${basedir}/src/main/proto</param>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml

@@ -73,7 +73,8 @@
               <goal>protoc</goal>
             </goals>
             <configuration>
-              <protocVersion>2.5.0</protocVersion>
+              <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
               <imports>
                 <param>${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
                 <param>${basedir}/../../hadoop-yarn-api/src/main/proto</param>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml

@@ -165,7 +165,8 @@
               <goal>protoc</goal>
             </goals>
             <configuration>
-              <protocVersion>2.5.0</protocVersion>
+              <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
               <imports>
                 <param>${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
                 <param>${basedir}/../../hadoop-yarn-api/src/main/proto</param>