MAPREDUCE-3039. Upgrade to Avro 1.5.3. Contributed by Joep Rottinghuis.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.22@1176694 13f79535-47bb-0310-9956-ffa450edef68

mapreduce/CHANGES.txt

@@ -223,6 +223,8 @@ Release 0.22.0 - Unreleased
     HADOOP-7106. Reorganize project SVN layout to "unsplit" the projects.
     (todd, nigel)
 
+    MAPREDUCE-3039. Upgrade to Avro 1.5.3. (Joep Rottinghuis via shv)
+
   OPTIMIZATIONS
     
     MAPREDUCE-1354. Enhancements to JobTracker for better performance and

mapreduce/build.xml

@@ -339,7 +339,7 @@
 
   <target name="avro-generate" depends="init">
     <mkdir dir="${build.src}/org/apache/hadoop/mapreduce/jobhistory"/>
-    <taskdef name="protocol" classname="org.apache.avro.specific.ProtocolTask">
+    <taskdef name="protocol" classname="org.apache.avro.compiler.specific.ProtocolTask">
       <classpath refid="classpath" />
     </taskdef>
     <protocol destdir="${build.src}">

mapreduce/ivy.xml

@@ -97,12 +97,18 @@
 
    <dependency org="org.apache.lucene" name="lucene-core" 
                rev="${lucene-core.version}" conf="javadoc->default"/>
-   <dependency org="org.apache.hadoop" name="avro" rev="${avro.version}" 
-               conf="common->default">
+   <dependency org="org.apache.avro" name="avro"
+               rev="${avro.version}" conf="common->default">
       <exclude module="ant"/>
       <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
+      <exclude module="slf4j-api"/>
+   </dependency>
+   <dependency org="org.apache.avro" name="avro-compiler"
+               rev="${avro.version}" conf="common->default">
+      <exclude module="ant"/>
+      <exclude module="jetty"/>
+      <exclude module="slf4j-api"/>
+   </dependency>
    <dependency org="org.mockito" name="mockito-all" rev="${mockito-all.version}" 
                conf="test->default"/>
    <dependency org="org.mortbay.jetty" name="jetty-servlet-tester" rev="${jetty.version}"

mapreduce/ivy/libraries.properties

@@ -20,7 +20,7 @@ ant-task.version=2.0.10
 #This property has to be updated synchronously with aop.xml
 aspectj.version=1.6.5
 
-avro.version=1.3.2
+avro.version=1.5.3
 
 checkstyle.version=4.2
 

mapreduce/src/contrib/capacity-scheduler/ivy.xml

@@ -69,11 +69,5 @@
                 rev="${servlet-api-2.5.version}" conf="common->master"/>
     <dependency org="commons-httpclient" name="commons-httpclient" 
                 rev="${commons-httpclient.version}" conf="common->master"/>
-    <dependency org="org.apache.hadoop" name="avro" 
-                rev="${avro.version}" conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
   </dependencies>
 </ivy-module>

mapreduce/src/contrib/fairscheduler/ivy.xml

@@ -64,26 +64,10 @@
       name="junit"
       rev="${junit.version}"
       conf="common->default"/>
-    <dependency org="org.apache.hadoop"
-      name="avro"
-      rev="${avro.version}"
-      conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
     <dependency org="org.codehaus.jackson"
       name="jackson-mapper-asl"
       rev="${jackson.version}"
       conf="common->default"/>
-    <dependency org="com.thoughtworks.paranamer"
-      name="paranamer"
-      rev="${paranamer.version}"
-      conf="common->default"/>
-    <dependency org="com.thoughtworks.paranamer"
-      name="paranamer-ant"
-      rev="${paranamer.version}"
-      conf="common->default"/>
     <dependency org="org.mortbay.jetty"
       name="jetty-util"
       rev="${jetty-util.version}"

mapreduce/src/contrib/gridmix/ivy.xml

@@ -102,14 +102,6 @@
       name="commons-cli"
       rev="${commons-cli.version}"
       conf="common->default"/>
-    <dependency org="org.apache.hadoop"
-      name="avro"
-      rev="${avro.version}"
-      conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
     <dependency org="org.codehaus.jackson"
       name="jackson-mapper-asl"
       rev="${jackson.version}"
@@ -118,9 +110,5 @@
       name="jackson-core-asl"
       rev="${jackson.version}"
       conf="common->default"/>
-    <dependency org="com.thoughtworks.paranamer"
-      name="paranamer"
-      rev="${paranamer.version}"
-      conf="common->default"/>
   </dependencies>
 </ivy-module>

mapreduce/src/contrib/mrunit/ivy.xml

@@ -45,11 +45,6 @@
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}" conf="common->default"/>
     <dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/>
     <dependency org="junit" name="junit" rev="${junit.version}" conf="common->master"/>
-    <dependency org="org.apache.hadoop" name="avro" rev="${avro.version}" conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
   </dependencies>
 
 </ivy-module>

mapreduce/src/contrib/mumak/ivy.xml

@@ -114,13 +114,5 @@
       name="commons-cli"
       rev="${commons-cli.version}"
       conf="common->default"/>
-    <dependency org="org.apache.hadoop"
-      name="avro"
-	  rev="${avro.version}"
-      conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
   </dependencies>
 </ivy-module>

mapreduce/src/contrib/raid/ivy.xml

@@ -102,14 +102,6 @@
       name="commons-cli"
       rev="${commons-cli.version}"
       conf="common->default"/>
-    <dependency org="org.apache.hadoop"
-      name="avro"
-      rev="${avro.version}"
-      conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
     <dependency org="org.codehaus.jackson"
       name="jackson-mapper-asl"
       rev="${jackson.version}"

mapreduce/src/contrib/streaming/ivy.xml

@@ -68,12 +68,6 @@
                 rev="${commons-httpclient.version}" conf="common->default"/>
     <dependency org="log4j" name="log4j" 
                 rev="${log4j.version}" conf="common->master"/>
-    <dependency org="org.apache.hadoop" name="avro" 
-                rev="${avro.version}" conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
     <dependency org="org.slf4j" name="slf4j-api" 
                 rev="${slf4j-api.version}" conf="common->master"/>
   </dependencies>

mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java

@@ -20,10 +20,14 @@ package org.apache.hadoop.mapreduce.jobhistory;
 
 import java.io.Closeable;
 import java.io.DataInputStream;
-import java.io.IOException;
 import java.io.EOFException;
-import java.io.StringBufferInputStream;
+import java.io.IOException;
 
+import org.apache.avro.Schema;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.specific.SpecificDatumReader;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FileSystem;
@@ -32,13 +36,6 @@ import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.CounterGroup;
 import org.apache.hadoop.mapreduce.Counters;
 
-import org.apache.avro.Schema;
-import org.apache.avro.AvroRuntimeException;
-import org.apache.avro.io.Decoder;
-import org.apache.avro.io.JsonDecoder;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.specific.SpecificDatumReader;
-
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class EventReader implements Closeable {
@@ -46,7 +43,7 @@ public class EventReader implements Closeable {
   private Schema schema;
   private DataInputStream in;
   private Decoder decoder;
-  private DatumReader reader;
+  private DatumReader<Event> reader;
 
   /**
    * Create a new Event Reader
@@ -73,8 +70,8 @@ public class EventReader implements Closeable {
     }
     
     this.schema = Schema.parse(in.readLine());
-    this.reader = new SpecificDatumReader(schema);
-    this.decoder = new JsonDecoder(schema, in);
+    this.reader = new SpecificDatumReader<Event>(schema);
+    this.decoder = DecoderFactory.get().jsonDecoder(schema, in);
   }
   
   /**
@@ -82,11 +79,10 @@ public class EventReader implements Closeable {
    * @return the next event
    * @throws IOException
    */
-  @SuppressWarnings("unchecked")
   public HistoryEvent getNextEvent() throws IOException {
     Event wrapper;
     try {
-      wrapper = (Event)reader.read(null, decoder);
+      wrapper = reader.read(null, decoder);
     } catch (EOFException e) {            // at EOF
       return null;
     }

mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java

@@ -19,20 +19,18 @@
 package org.apache.hadoop.mapreduce.jobhistory;
 
 import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.CounterGroup;
-import org.apache.hadoop.mapreduce.Counters;
 
 import org.apache.avro.Schema;
-import org.apache.avro.io.Encoder;
-import org.apache.avro.io.JsonEncoder;
+import org.apache.avro.generic.GenericData;
 import org.apache.avro.io.DatumWriter;
+import org.apache.avro.io.Encoder;
+import org.apache.avro.io.EncoderFactory;
 import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.avro.generic.GenericData;
 import org.apache.avro.util.Utf8;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
 
 /**
  * Event Writer is an utility class used to write events to the underlying
@@ -54,7 +52,7 @@ class EventWriter {
     out.writeBytes("\n");
     out.writeBytes(Event.SCHEMA$.toString());
     out.writeBytes("\n");
-    this.encoder = new JsonEncoder(Event.SCHEMA$, out);
+    this.encoder = EncoderFactory.get().jsonEncoder(Event.SCHEMA$, out);
   }
   
   synchronized void write(HistoryEvent event) throws IOException { 

mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java

@@ -57,7 +57,7 @@ public class JobSubmittedEvent implements HistoryEvent {
     datum.userName = new Utf8(userName);
     datum.submitTime = submitTime;
     datum.jobConfPath = new Utf8(jobConfPath);
-    Map<Utf8, Utf8> jobAcls = new HashMap<Utf8, Utf8>();
+    Map<CharSequence, CharSequence> jobAcls = new HashMap<CharSequence, CharSequence>();
     for (Entry<JobACL, AccessControlList> entry : jobACLs.entrySet()) {
       jobAcls.put(new Utf8(entry.getKey().getAclName()), new Utf8(
           entry.getValue().getAclString()));