HADOOP-941. Enhance record facility. Contributed by Milind.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@513122 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -168,6 +168,9 @@ Trunk (unreleased changes)
 50. HADOOP-1020.  Fix a bug in Path resolution, and a with unit tests
     on Windows.  (cutting)
 
+51. HADOOP-941.  Enhance record facility.
+    (Milind Bhandarkar via cutting)
+
 
 Release 0.11.2 - 2007-02-16
 

src/c++/librecordio/binarchive.cc

@@ -17,6 +17,8 @@
  */
 
 #include "binarchive.hh"
+#include <rpc/xdr.h>
+
 
 using namespace hadoop;
 
@@ -77,16 +79,20 @@ static void deserializeInt(int32_t& t, InStream& stream)
     t = b;
     return;
   }
-  b = (b < -124) ? -(b + 124) : -(b + 120);
+  bool isNegative = (b < -124);
+  b = isNegative ? -(b + 124) : -(b + 120);
   uint8_t barr[b];
   if (b != stream.read(barr, b)) {
-    throw new IOException("Error deserializing long");
+    throw new IOException("Error deserializing int");
   }
   t = 0;
   for (int idx = 0; idx < b; idx++) {
     t = t << 8;
     t |= (barr[idx] & 0xFF);
   }
+  if (isNegative) {
+    t |= 0x80000000;
+  }
 }
 
 static void serializeLong(int64_t t, OutStream& stream)
@@ -131,7 +137,8 @@ static void deserializeLong(int64_t& t, InStream& stream)
     t = b;
     return;
   }
-  b = (b < -120) ? -(b + 120) : -(b + 112);
+  bool isNegative = (b < -120);
+  b = isNegative ? -(b + 120) : -(b + 112);
   uint8_t barr[b];
   if (b != stream.read(barr, b)) {
     throw new IOException("Error deserializing long.");
@@ -141,6 +148,9 @@ static void deserializeLong(int64_t& t, InStream& stream)
     t = t << 8;
     t |= (barr[idx] & 0xFF);
   }
+  if (isNegative) {
+    t |= 0x8000000000000000L;
+  }
 }
 
 static void serializeFloat(float t, OutStream& stream)

src/c++/librecordio/binarchive.hh

@@ -20,7 +20,6 @@
 #define BINARCHIVE_HH_
 
 #include "recordio.hh"
-#include <rpc/xdr.h>
 
 namespace hadoop {
 

src/java/org/apache/hadoop/io/WritableComparator.java

@@ -177,7 +177,8 @@ public class WritableComparator implements Comparator {
       if (len >= -112) {
           return len;
       }
-      len = (len < -120) ? -(len + 120) : -(len + 112);
+      boolean isNegative = (len < -120);
+      len = isNegative ? -(len + 120) : -(len + 112);
       if (start+1+len>bytes.length)
           throw new IOException(
                   "Not enough number of bytes for a zero-compressed integer");
@@ -186,7 +187,7 @@ public class WritableComparator implements Comparator {
           i = i << 8;
           i = i | (bytes[start+1+idx] & 0xFF);
       }
-      return i;
+      return (isNegative ? (i | 0x8000000000000000L) : i);
   }
   
   /**

src/java/org/apache/hadoop/io/WritableUtils.java

@@ -312,15 +312,15 @@ public final class WritableUtils  {
       if (len >= -112) {
           return len;
       }
-      len = (len < -120) ? -(len + 120) : -(len + 112);
-      byte[] barr = new byte[len];
-      stream.readFully(barr);
+      boolean isNegative = (len < -120);
+      len = isNegative ? -(len + 120) : -(len + 112);
       long i = 0;
       for (int idx = 0; idx < len; idx++) {
+          byte b = stream.readByte();
           i = i << 8;
-          i = i | (barr[idx] & 0xFF);
+          i = i | (b & 0xFF);
       }
-      return i;
+      return (isNegative ? (i | 0x8000000000000000L) : i);
   }
 
   /**

src/java/org/apache/hadoop/record/BinaryInputArchive.java

@@ -22,11 +22,6 @@ import java.io.DataInput;
 import java.io.IOException;
 import java.io.DataInputStream;
 import java.io.InputStream;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
-
-import org.apache.hadoop.io.WritableUtils;
-
 
 /**
  *
@@ -34,7 +29,7 @@ import org.apache.hadoop.io.WritableUtils;
  */
 public class BinaryInputArchive implements InputArchive {
     
-    private DataInput in;
+    final private DataInput in;
     
     static BinaryInputArchive getArchive(InputStream strm) {
         return new BinaryInputArchive(new DataInputStream(strm));
@@ -42,8 +37,8 @@ public class BinaryInputArchive implements InputArchive {
     
     static private class BinaryIndex implements Index {
         private int nelems;
-        BinaryIndex(int nelems) {
-            this.nelems = nelems;
+        private BinaryIndex(int nelems) {
+          this.nelems = nelems;
         }
         public boolean done() {
             return (nelems <= 0);
@@ -57,61 +52,69 @@ public class BinaryInputArchive implements InputArchive {
         this.in = in;
     }
     
-    public byte readByte(String tag) throws IOException {
+    public byte readByte(final String tag) throws IOException {
         return in.readByte();
     }
     
-    public boolean readBool(String tag) throws IOException {
+    public boolean readBool(final String tag) throws IOException {
         return in.readBoolean();
     }
     
-    public int readInt(String tag) throws IOException {
-        return WritableUtils.readVInt(in);
+    public int readInt(final String tag) throws IOException {
+        return Utils.readVInt(in);
     }
     
-    public long readLong(String tag) throws IOException {
-        return WritableUtils.readVLong(in);
+    public long readLong(final String tag) throws IOException {
+        return Utils.readVLong(in);
     }
     
-    public float readFloat(String tag) throws IOException {
+    public float readFloat(final String tag) throws IOException {
         return in.readFloat();
     }
     
-    public double readDouble(String tag) throws IOException {
+    public double readDouble(final String tag) throws IOException {
         return in.readDouble();
     }
     
-    public Text readString(String tag) throws IOException {
-        Text text = new Text();
-        text.readFields(in);
-        return text;
+    public String readString(final String tag) throws IOException {
+      final int length = Utils.readVInt(in);
+      final byte[] bytes = new byte[length];
+      in.readFully(bytes);
+      return new String(bytes, "UTF-8");
     }
     
-    public BytesWritable readBuffer(String tag) throws IOException {
-      int len = WritableUtils.readVInt(in);
-      byte[] barr = new byte[len];
+    public Buffer readBuffer(final String tag) throws IOException {
+      final int len = Utils.readVInt(in);
+      final byte[] barr = new byte[len];
       in.readFully(barr);
-      return new BytesWritable(barr);
+      return new Buffer(barr);
     }
     
-    public void readRecord(Record r, String tag) throws IOException {
-        r.deserialize(this, tag);
+    public void readRecord(final Record record, final String tag) throws IOException {
+        record.deserialize(this, tag);
     }
     
-    public void startRecord(String tag) throws IOException {}
+    public void startRecord(final String tag) throws IOException {
+      // no-op
+    }
     
-    public void endRecord(String tag) throws IOException {}
+    public void endRecord(final String tag) throws IOException {
+      // no-op
+    }
     
-    public Index startVector(String tag) throws IOException {
-        return new BinaryIndex(readInt(tag));
+    public Index startVector(final String tag) throws IOException {
+      return new BinaryIndex(readInt(tag));
     }
     
-    public void endVector(String tag) throws IOException {}
+    public void endVector(final String tag) throws IOException {
+      // no-op
+}
     
-    public Index startMap(String tag) throws IOException {
+    public Index startMap(final String tag) throws IOException {
         return new BinaryIndex(readInt(tag));
     }
     
-    public void endMap(String tag) throws IOException {}
-    
+    public void endMap(final String tag) throws IOException {
+      // no-op
+    }
 }

src/java/org/apache/hadoop/record/BinaryOutputArchive.java

@@ -24,10 +24,6 @@ import java.util.ArrayList;
 import java.io.DataOutput;
 import java.io.DataOutputStream;
 import java.io.OutputStream;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
-
-import org.apache.hadoop.io.WritableUtils;
 
 /**
  *
@@ -55,11 +51,11 @@ public class BinaryOutputArchive implements OutputArchive {
     }
     
     public void writeInt(int i, String tag) throws IOException {
-        WritableUtils.writeVInt(out, i);
+        Utils.writeVInt(out, i);
     }
     
     public void writeLong(long l, String tag) throws IOException {
-        WritableUtils.writeVLong(out, l);
+        Utils.writeVLong(out, l);
     }
     
     public void writeFloat(float f, String tag) throws IOException {
@@ -70,15 +66,18 @@ public class BinaryOutputArchive implements OutputArchive {
         out.writeDouble(d);
     }
     
-    public void writeString(Text s, String tag) throws IOException {
-        s.write(out);
+    public void writeString(String s, String tag) throws IOException {
+      byte[] bytes = s.getBytes("UTF-8");
+      int length = bytes.length;
+      Utils.writeVInt(out, length);
+      out.write(bytes);
     }
     
-    public void writeBuffer(BytesWritable buf, String tag)
+    public void writeBuffer(Buffer buf, String tag)
     throws IOException {
       byte[] barr = buf.get();
-      int len = buf.getSize();
-      WritableUtils.writeVInt(out, len);
+      int len = buf.getCount();
+      Utils.writeVInt(out, len);
       out.write(barr, 0, len);
     }
     

src/java/org/apache/hadoop/record/Buffer.java

@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.UnsupportedEncodingException;
+
+/**
+ * A byte sequence that is used as a Java native type for buffer.
+ * It is resizable and distinguishes between the count of the seqeunce and
+ * the current capacity.
+ * 
+ * @author Milind Bhandarkar
+ */
+public class Buffer implements Comparable, Cloneable {
+  /** Number of valid bytes in this.bytes. */
+  private int count;
+  /** Backing store for Buffer. */
+  private byte[] bytes = null;
+
+  /**
+   * Create a zero-count sequence.
+   */
+  public Buffer() {
+    this.count = 0;
+  }
+
+  /**
+   * Create a Buffer using the byte array as the initial value.
+   *
+   * @param bytes This array becomes the backing storage for the object.
+   */
+  public Buffer(byte[] bytes) {
+    this.bytes = bytes;
+    this.count = (bytes == null) ? 0 : bytes.length;
+  }
+  
+  /**
+   * Create a Buffer using the byte range as the initial value.
+   *
+   * @param bytes Copy of this array becomes the backing storage for the object.
+   * @param offset offset into byte array
+   * @param length length of data
+   */
+  public Buffer(byte[] bytes, int offset, int length) {
+    copy(bytes, offset, length);
+  }
+  
+  
+  /**
+   * Use the specified bytes array as underlying sequence.
+   *
+   * @param bytes byte sequence
+   */
+  public void set(byte[] bytes) {
+    this.count = (bytes == null) ? 0 : bytes.length;
+    this.bytes = bytes;
+  }
+  
+  /**
+   * Copy the specified byte array to the Buffer. Replaces the current buffer.
+   *
+   * @param bytes byte array to be assigned
+   * @param offset offset into byte array
+   * @param length length of data
+   */
+  public final void copy(byte[] bytes, int offset, int length) {
+    if (this.bytes == null || this.bytes.length < length) {
+      this.bytes = new byte[length];
+    }
+    System.arraycopy(bytes, offset, this.bytes, 0, length);
+    this.count = length;
+  }
+  
+  /**
+   * Get the data from the Buffer.
+   * 
+   * @return The data is only valid between 0 and getCount() - 1.
+   */
+  public byte[] get() {
+    if (bytes == null) {
+      bytes = new byte[0];
+    }
+    return bytes;
+  }
+  
+  /**
+   * Get the current count of the buffer.
+   */
+  public int getCount() {
+    return count;
+  }
+  
+  /**
+   * Get the capacity, which is the maximum count that could handled without
+   * resizing the backing storage.
+   * 
+   * @return The number of bytes
+   */
+  public int getCapacity() {
+    return this.get().length;
+  }
+  
+  /**
+   * Change the capacity of the backing storage.
+   * The data is preserved if newCapacity >= getCount().
+   * @param newCapacity The new capacity in bytes.
+   */
+  public void setCapacity(int newCapacity) {
+    if (newCapacity < 0) {
+     throw new IllegalArgumentException("Invalid capacity argument "+newCapacity); 
+    }
+    if (newCapacity == 0) {
+      this.bytes = null;
+      this.count = 0;
+      return;
+    }
+    if (newCapacity != getCapacity()) {
+      byte[] data = new byte[newCapacity];
+      if (newCapacity < count) {
+        count = newCapacity;
+      }
+      if (count != 0) {
+        System.arraycopy(this.get(), 0, data, 0, count);
+      }
+      bytes = data;
+    }
+  }
+  
+  /**
+   * Reset the buffer to 0 size
+   */
+  public void reset() {
+    setCapacity(0);
+  }
+  
+  /**
+   * Change the capacity of the backing store to be the same as the current 
+   * count of buffer.
+   */
+  public void truncate() {
+    setCapacity(count);
+  }
+  
+  /**
+   * Append specified bytes to the buffer.
+   *
+   * @param bytes byte array to be appended
+   * @param offset offset into byte array
+   * @param length length of data
+
+   */
+  public void append(byte[] bytes, int offset, int length) {
+    setCapacity(count+length);
+    System.arraycopy(bytes, offset, this.get(), count, length);
+    count = count + length;
+  }
+  
+  /**
+   * Append specified bytes to the buffer
+   *
+   * @param bytes byte array to be appended
+   */
+  public void append(byte[] bytes) {
+    append(bytes, 0, bytes.length);
+  }
+  
+  // inherit javadoc
+  public int hashCode() {
+    int hash = 1;
+    byte[] b = this.get();
+    for (int i = 0; i < count; i++)
+      hash = (31 * hash) + (int)b[i];
+    return hash;
+  }
+  
+  /**
+   * Define the sort order of the Buffer.
+   * 
+   * @param other The other buffer
+   * @return Positive if this is bigger than other, 0 if they are equal, and
+   *         negative if this is smaller than other.
+   */
+  public int compareTo(Object other) {
+    Buffer right = ((Buffer) other);
+    byte[] lb = this.get();
+    byte[] rb = right.get();
+    for (int i = 0; i < count && i < right.count; i++) {
+      int a = (lb[i] & 0xff);
+      int b = (rb[i] & 0xff);
+      if (a != b) {
+        return a - b;
+      }
+    }
+    return count - right.count;
+  }
+  
+  // inherit javadoc
+  public boolean equals(Object other) {
+    if (other instanceof Buffer && this != other) {
+      return compareTo(other) == 0;
+    }
+    return (this == other);
+  }
+  
+  // inheric javadoc
+  public String toString() {
+    StringBuffer sb = new StringBuffer(2*count);
+    for(int idx = 0; idx < count; idx++) {
+      sb.append(Integer.toHexString((int)bytes[idx]));
+    }
+    return sb.toString();
+  }
+  
+  /**
+   * Convert the byte buffer to a string an specific character encoding
+   *
+   * @param charsetName Valid Java Character Set Name
+   */
+  public String toString(String charsetName)
+  throws UnsupportedEncodingException {
+    return new String(this.get(), 0, this.getCount(), charsetName);
+  }
+  
+  // inherit javadoc
+  public Object clone() throws CloneNotSupportedException {
+    return new Buffer(this.get().clone(), 0, this.getCount());
+  }
+}

src/java/org/apache/hadoop/record/CsvInputArchive.java

@@ -23,8 +23,6 @@ import java.io.InputStream;
 import java.io.IOException;
 import java.io.PushbackReader;
 import java.io.UnsupportedEncodingException;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  *
@@ -121,13 +119,12 @@ class CsvInputArchive implements InputArchive {
         }
     }
     
-    public Text readString(String tag) throws IOException {
+    public String readString(String tag) throws IOException {
         String sval = readField(tag);
         return Utils.fromCSVString(sval);
-        
     }
     
-    public BytesWritable readBuffer(String tag) throws IOException {
+    public Buffer readBuffer(String tag) throws IOException {
         String sval = readField(tag);
         return Utils.fromCSVBuffer(sval);
     }

src/java/org/apache/hadoop/record/CsvOutputArchive.java

@@ -24,8 +24,6 @@ import java.util.ArrayList;
 import java.io.PrintStream;
 import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  *
@@ -91,13 +89,13 @@ public class CsvOutputArchive implements OutputArchive {
         throwExceptionOnError(tag);
     }
     
-    public void writeString(Text s, String tag) throws IOException {
+    public void writeString(String s, String tag) throws IOException {
         printCommaUnlessFirst();
         stream.print(Utils.toCSVString(s));
         throwExceptionOnError(tag);
     }
     
-    public void writeBuffer(BytesWritable buf, String tag)
+    public void writeBuffer(Buffer buf, String tag)
     throws IOException {
         printCommaUnlessFirst();
         stream.print(Utils.toCSVBuffer(buf));

src/java/org/apache/hadoop/record/Index.java

@@ -34,6 +34,6 @@ package org.apache.hadoop.record;
  * @author Milind Bhandarkar
  */
 public interface Index {
-    public boolean done();
-    public void incr();
+    boolean done();
+    void incr();
 }

src/java/org/apache/hadoop/record/InputArchive.java

@@ -19,8 +19,6 @@
 package org.apache.hadoop.record;
 
 import java.io.IOException;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  * Interface that all the Deserializers have to implement.
@@ -34,8 +32,8 @@ public interface InputArchive {
     public long readLong(String tag) throws IOException;
     public float readFloat(String tag) throws IOException;
     public double readDouble(String tag) throws IOException;
-    public Text readString(String tag) throws IOException;
-    public BytesWritable readBuffer(String tag) throws IOException;
+    public String readString(String tag) throws IOException;
+    public Buffer readBuffer(String tag) throws IOException;
     public void readRecord(Record r, String tag) throws IOException;
     public void startRecord(String tag) throws IOException;
     public void endRecord(String tag) throws IOException;

src/java/org/apache/hadoop/record/OutputArchive.java

@@ -21,8 +21,6 @@ package org.apache.hadoop.record;
 import java.io.IOException;
 import java.util.TreeMap;
 import java.util.ArrayList;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  * Interface that alll the serializers have to implement.
@@ -36,8 +34,8 @@ public interface OutputArchive {
     public void writeLong(long l, String tag) throws IOException;
     public void writeFloat(float f, String tag) throws IOException;
     public void writeDouble(double d, String tag) throws IOException;
-    public void writeString(Text s, String tag) throws IOException;
-    public void writeBuffer(BytesWritable buf, String tag)
+    public void writeString(String s, String tag) throws IOException;
+    public void writeBuffer(Buffer buf, String tag)
         throws IOException;
     public void writeRecord(Record r, String tag) throws IOException;
     public void startRecord(Record r, String tag) throws IOException;

src/java/org/apache/hadoop/record/Record.java

@@ -25,9 +25,9 @@ import java.io.IOException;
  * 
  * @author Milind Bhandarkar
  */
-public interface Record {
-    public void serialize(OutputArchive archive, String tag)
+public interface Record extends Cloneable {
+    void serialize(OutputArchive archive, String tag)
         throws IOException;
-    public void deserialize(InputArchive archive, String tag)
+    void deserialize(InputArchive archive, String tag)
         throws IOException;
 }

src/java/org/apache/hadoop/record/RecordReader.java

@@ -20,13 +20,9 @@ package org.apache.hadoop.record;
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.DataInputStream;
-import java.io.UnsupportedEncodingException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.HashMap;
-import javax.xml.parsers.ParserConfigurationException;
-import org.xml.sax.SAXException;
 
 /**
  * Front-end interface to deserializers. Also acts as a factory

src/java/org/apache/hadoop/record/RecordWriter.java

@@ -21,13 +21,10 @@ package org.apache.hadoop.record;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.DataOutputStream;
-import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.HashMap;
-import javax.xml.parsers.ParserConfigurationException;
-import org.xml.sax.SAXException;
 
 /**
  * Front-end for serializers. Also serves as a factory for serializers.

src/java/org/apache/hadoop/record/Utils.java

@@ -18,224 +18,316 @@
 
 package org.apache.hadoop.record;
 
-import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataOutput;
 import java.io.IOException;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.WritableUtils;
 
 /**
  * Various utility functions for Hadooop record I/O runtime.
  * @author Milind Bhandarkar
  */
 public class Utils {
-    
-    /** Cannot create a new instance of Utils */
-    private Utils() {
+  
+  /** Cannot create a new instance of Utils */
+  private Utils() {
+  }
+  
+  public static final char[] hexchars = { '0', '1', '2', '3', '4', '5',
+  '6', '7', '8', '9', 'A', 'B',
+  'C', 'D', 'E', 'F' };
+  /**
+   *
+   * @param s
+   * @return
+   */
+  static String toXMLString(String s) {
+    StringBuffer sb = new StringBuffer();
+    for (int idx = 0; idx < s.length(); idx++) {
+      char ch = s.charAt(idx);
+      if (ch == '<') {
+        sb.append("&lt;");
+      } else if (ch == '&') {
+        sb.append("&amp;");
+      } else if (ch == '%') {
+        sb.append("%25");
+      } else if (ch < 0x20) {
+        sb.append("%");
+        sb.append(hexchars[ch/16]);
+        sb.append(hexchars[ch%16]);
+      } else {
+        sb.append(ch);
+      }
     }
-   
-    public static final char[] hexchars = { '0', '1', '2', '3', '4', '5',
-                                            '6', '7', '8', '9', 'A', 'B',
-                                            'C', 'D', 'E', 'F' };
-    /**
-     * 
-     * @param s 
-     * @return 
-     */
-    static String toXMLString(Text t) {
-        String s = t.toString();
-        StringBuffer sb = new StringBuffer();
-        for (int idx = 0; idx < s.length(); idx++) {
-          char ch = s.charAt(idx);
-          if (ch == '<') {
-            sb.append("&lt;");
-          } else if (ch == '&') {
-            sb.append("&amp;");
-          } else if (ch == '%') {
-            sb.append("%25");
-          } else if (ch < 0x20) {
-            sb.append("%");
-            sb.append(hexchars[ch/16]);
-            sb.append(hexchars[ch%16]);
-          } else {
-            sb.append(ch);
-          }
-        }
-        return sb.toString();
+    return sb.toString();
+  }
+  
+  static private int h2c(char ch) {
+    if (ch >= '0' && ch <= '9') {
+      return ch - '0';
+    } else if (ch >= 'A' && ch <= 'F') {
+      return ch - 'A';
+    } else if (ch >= 'a' && ch <= 'f') {
+      return ch - 'a';
     }
-    
-    static private int h2c(char ch) {
-      if (ch >= '0' && ch <= '9') {
-        return ch - '0';
-      } else if (ch >= 'A' && ch <= 'F') {
-        return ch - 'A';
-      } else if (ch >= 'a' && ch <= 'f') {
-        return ch - 'a';
+    return 0;
+  }
+  
+  /**
+   *
+   * @param s
+   * @return
+   */
+  static String fromXMLString(String s) {
+    StringBuffer sb = new StringBuffer();
+    for (int idx = 0; idx < s.length();) {
+      char ch = s.charAt(idx++);
+      if (ch == '%') {
+        char ch1 = s.charAt(idx++);
+        char ch2 = s.charAt(idx++);
+        char res = (char)(h2c(ch1)*16 + h2c(ch2));
+        sb.append(res);
+      } else {
+        sb.append(ch);
       }
-      return 0;
     }
     
-    /**
-     * 
-     * @param s 
-     * @return 
-     */
-    static Text fromXMLString(String s) {
-        StringBuffer sb = new StringBuffer();
-        for (int idx = 0; idx < s.length();) {
-          char ch = s.charAt(idx++);
-          if (ch == '%') {
-            char ch1 = s.charAt(idx++);
-            char ch2 = s.charAt(idx++);
-            char res = (char)(h2c(ch1)*16 + h2c(ch2));
-            sb.append(res);
-          } else {
-            sb.append(ch);
-          }
-        }
-        
-        return new Text(sb.toString());
+    return sb.toString();
+  }
+  
+  /**
+   *
+   * @param s
+   * @return
+   */
+  static String toCSVString(String s) {
+    StringBuffer sb = new StringBuffer(s.length()+1);
+    sb.append('\'');
+    int len = s.length();
+    for (int i = 0; i < len; i++) {
+      char c = s.charAt(i);
+      switch(c) {
+        case '\0':
+          sb.append("%00");
+          break;
+        case '\n':
+          sb.append("%0A");
+          break;
+        case '\r':
+          sb.append("%0D");
+          break;
+        case ',':
+          sb.append("%2C");
+          break;
+        case '}':
+          sb.append("%7D");
+          break;
+        case '%':
+          sb.append("%25");
+          break;
+        default:
+          sb.append(c);
+      }
     }
-    
-    /**
-     * 
-     * @param s 
-     * @return 
-     */
-    static String toCSVString(Text t) {
-        String s = t.toString();
-        StringBuffer sb = new StringBuffer(s.length()+1);
-        sb.append('\'');
-        int len = s.length();
-        for (int i = 0; i < len; i++) {
-            char c = s.charAt(i);
-            switch(c) {
-                case '\0':
-                    sb.append("%00");
-                    break;
-                case '\n':
-                    sb.append("%0A");
-                    break;
-                case '\r':
-                    sb.append("%0D");
-                    break;
-                case ',':
-                    sb.append("%2C");
-                    break;
-                case '}':
-                    sb.append("%7D");
-                    break;
-                case '%':
-                    sb.append("%25");
-                    break;
-                default:
-                    sb.append(c);
-            }
-        }
-        return sb.toString();
+    return sb.toString();
+  }
+  
+  /**
+   *
+   * @param s
+   * @throws java.io.IOException
+   * @return
+   */
+  static String fromCSVString(String s) throws IOException {
+    if (s.charAt(0) != '\'') {
+      throw new IOException("Error deserializing string.");
     }
-    
-    /**
-     * 
-     * @param s 
-     * @throws java.io.IOException 
-     * @return 
-     */
-    static Text fromCSVString(String s) throws IOException {
-        if (s.charAt(0) != '\'') {
-            throw new IOException("Error deserializing string.");
+    int len = s.length();
+    StringBuffer sb = new StringBuffer(len-1);
+    for (int i = 1; i < len; i++) {
+      char c = s.charAt(i);
+      if (c == '%') {
+        char ch1 = s.charAt(i+1);
+        char ch2 = s.charAt(i+2);
+        i += 2;
+        if (ch1 == '0' && ch2 == '0') {
+          sb.append('\0');
+        } else if (ch1 == '0' && ch2 == 'A') {
+          sb.append('\n');
+        } else if (ch1 == '0' && ch2 == 'D') {
+          sb.append('\r');
+        } else if (ch1 == '2' && ch2 == 'C') {
+          sb.append(',');
+        } else if (ch1 == '7' && ch2 == 'D') {
+          sb.append('}');
+        } else if (ch1 == '2' && ch2 == '5') {
+          sb.append('%');
+        } else {
+          throw new IOException("Error deserializing string.");
         }
-        int len = s.length();
-        StringBuffer sb = new StringBuffer(len-1);
-        for (int i = 1; i < len; i++) {
-            char c = s.charAt(i);
-            if (c == '%') {
-                char ch1 = s.charAt(i+1);
-                char ch2 = s.charAt(i+2);
-                i += 2;
-                if (ch1 == '0' && ch2 == '0') { sb.append('\0'); }
-                else if (ch1 == '0' && ch2 == 'A') { sb.append('\n'); }
-                else if (ch1 == '0' && ch2 == 'D') { sb.append('\r'); }
-                else if (ch1 == '2' && ch2 == 'C') { sb.append(','); }
-                else if (ch1 == '7' && ch2 == 'D') { sb.append('}'); }
-                else if (ch1 == '2' && ch2 == '5') { sb.append('%'); }
-                else {throw new IOException("Error deserializing string.");}
-            } else {
-                sb.append(c);
-            }
-        }
-        return new Text(sb.toString());
-    }
-    
-    /**
-     * 
-     * @param s 
-     * @return 
-     */
-    static String toXMLBuffer(BytesWritable s) {
-        byte[] barr = s.get();
-        int bsize = s.getSize();
-        StringBuffer sb = new StringBuffer(2*bsize);
-        for (int idx = 0; idx < bsize; idx++) {
-            sb.append(Integer.toHexString((int)barr[idx]));
-        }
-        return sb.toString();
+      } else {
+        sb.append(c);
+      }
     }
-    
-    /**
-     * 
-     * @param s 
-     * @throws java.io.IOException 
-     * @return 
-     */
-    static BytesWritable fromXMLBuffer(String s)
-    throws IOException {
-        if (s.length() == 0) { return new BytesWritable(); }
-        int blen = s.length()/2;
-        byte[] barr = new byte[blen];
-        for (int idx = 0; idx < blen; idx++) {
-            char c1 = s.charAt(2*idx);
-            char c2 = s.charAt(2*idx+1);
-            barr[idx] = Byte.parseByte(""+c1+c2, 16);
-        }
-        return new BytesWritable(barr);
+    return sb.toString();
+  }
+  
+  /**
+   *
+   * @param s
+   * @return
+   */
+  static String toXMLBuffer(Buffer s) {
+    return s.toString();
+  }
+  
+  /**
+   *
+   * @param s
+   * @throws java.io.IOException
+   * @return
+   */
+  static Buffer fromXMLBuffer(String s)
+  throws IOException {
+    if (s.length() == 0) { return new Buffer(); }
+    int blen = s.length()/2;
+    byte[] barr = new byte[blen];
+    for (int idx = 0; idx < blen; idx++) {
+      char c1 = s.charAt(2*idx);
+      char c2 = s.charAt(2*idx+1);
+      barr[idx] = Byte.parseByte(""+c1+c2, 16);
     }
-    
-    /**
-     * 
-     * @param buf 
-     * @return 
-     */
-    static String toCSVBuffer(BytesWritable buf) {
-        byte[] barr = buf.get();
-        int bsize = buf.getSize();
-        StringBuffer sb = new StringBuffer(bsize+1);
-        sb.append('#');
-        for(int idx = 0; idx < bsize; idx++) {
-            sb.append(Integer.toHexString((int)barr[idx]));
-        }
-        return sb.toString();
+    return new Buffer(barr);
+  }
+  
+  /**
+   *
+   * @param buf
+   * @return
+   */
+  static String toCSVBuffer(Buffer buf) {
+    StringBuffer sb = new StringBuffer("#");
+    sb.append(buf.toString());
+    return sb.toString();
+  }
+  
+  /**
+   * Converts a CSV-serialized representation of buffer to a new
+   * Buffer
+   * @param s CSV-serialized representation of buffer
+   * @throws java.io.IOException
+   * @return Deserialized Buffer
+   */
+  static Buffer fromCSVBuffer(String s)
+  throws IOException {
+    if (s.charAt(0) != '#') {
+      throw new IOException("Error deserializing buffer.");
     }
-    
-    /**
-     * Converts a CSV-serialized representation of buffer to a new
-     * BytesWritable.
-     * @param s CSV-serialized representation of buffer
-     * @throws java.io.IOException 
-     * @return Deserialized BytesWritable
-     */
-    static BytesWritable fromCSVBuffer(String s)
-    throws IOException {
-        if (s.charAt(0) != '#') {
-            throw new IOException("Error deserializing buffer.");
-        }
-        if (s.length() == 1) { return new BytesWritable(); }
-        int blen = (s.length()-1)/2;
-        byte[] barr = new byte[blen];
-        for (int idx = 0; idx < blen; idx++) {
-            char c1 = s.charAt(2*idx+1);
-            char c2 = s.charAt(2*idx+2);
-            barr[idx] = Byte.parseByte(""+c1+c2, 16);
-        }
-        return new BytesWritable(barr);
+    if (s.length() == 1) { return new Buffer(); }
+    int blen = (s.length()-1)/2;
+    byte[] barr = new byte[blen];
+    for (int idx = 0; idx < blen; idx++) {
+      char c1 = s.charAt(2*idx+1);
+      char c2 = s.charAt(2*idx+2);
+      barr[idx] = Byte.parseByte(""+c1+c2, 16);
     }
+    return new Buffer(barr);
+  }
+  
+  /** Parse a float from a byte array. */
+  public static float readFloat(byte[] bytes, int start) {
+    return WritableComparator.readFloat(bytes, start);
+  }
+  
+  /** Parse a double from a byte array. */
+  public static double readDouble(byte[] bytes, int start) {
+    return WritableComparator.readDouble(bytes, start);
+  }
+  
+  /**
+   * Reads a zero-compressed encoded long from a byte array and returns it.
+   * @param bytes byte array with decode long
+   * @param start starting index
+   * @throws java.io.IOException
+   * @return deserialized long
+   */
+  public static long readVLong(byte[] bytes, int start) throws IOException {
+    return WritableComparator.readVLong(bytes, start);
+  }
+  
+  /**
+   * Reads a zero-compressed encoded integer from a byte array and returns it.
+   * @param bytes byte array with the encoded integer
+   * @param start start index
+   * @throws java.io.IOException
+   * @return deserialized integer
+   */
+  public static int readVInt(byte[] bytes, int start) throws IOException {
+    return WritableComparator.readVInt(bytes, start);
+  }
+  
+  /**
+   * Reads a zero-compressed encoded long from a stream and return it.
+   * @param in input stream
+   * @throws java.io.IOException
+   * @return deserialized long
+   */
+  public static long readVLong(DataInput in) throws IOException {
+    return WritableUtils.readVLong(in);
+  }
+  
+  /**
+   * Reads a zero-compressed encoded integer from a stream and returns it.
+   * @param in input stream
+   * @throws java.io.IOException
+   * @return deserialized integer
+   */
+  public static int readVInt(DataInput in) throws IOException {
+    return WritableUtils.readVInt(in);
+  }
+  
+  /**
+   * Get the encoded length if an integer is stored in a variable-length format
+   * @return the encoded length
+   */
+  public static int getVIntSize(long i) {
+    return WritableUtils.getVIntSize(i);
+  }
+  
+  /**
+   * Serializes a long to a binary stream with zero-compressed encoding.
+   * For -112 <= i <= 127, only one byte is used with the actual value.
+   * For other values of i, the first byte value indicates whether the
+   * long is positive or negative, and the number of bytes that follow.
+   * If the first byte value v is between -113 and -120, the following long
+   * is positive, with number of bytes that follow are -(v+112).
+   * If the first byte value v is between -121 and -128, the following long
+   * is negative, with number of bytes that follow are -(v+120). Bytes are
+   * stored in the high-non-zero-byte-first order.
+   *
+   * @param stream Binary output stream
+   * @param i Long to be serialized
+   * @throws java.io.IOException
+   */
+  public static void writeVLong(DataOutput stream, long i) throws IOException {
+    WritableUtils.writeVLong(stream, i);
+  }
+  
+  /**
+   * Serializes an int to a binary stream with zero-compressed encoding.
+   *
+   * @param stream Binary output stream
+   * @param i int to be serialized
+   * @throws java.io.IOException
+   */
+  public static void writeVInt(DataOutput stream, int i) throws IOException {
+    WritableUtils.writeVInt(stream, i);
+  }
+  
+  /** Lexicographic order of binary data. */
+  public static int compareBytes(byte[] b1, int s1, int l1,
+      byte[] b2, int s2, int l2) {
+    return WritableComparator.compareBytes(b1, s1, l1, b2, s2, l2);
+  }
 }

src/java/org/apache/hadoop/record/XmlInputArchive.java

@@ -27,8 +27,7 @@ import org.xml.sax.helpers.DefaultHandler;
 import javax.xml.parsers.SAXParserFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParser;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
+
 /**
  *
  * @author Milind Bhandarkar
@@ -199,7 +198,7 @@ class XmlInputArchive implements InputArchive {
         return Double.parseDouble(v.getValue());
     }
     
-    public Text readString(String tag) throws IOException {
+    public String readString(String tag) throws IOException {
         Value v = next();
         if (!"string".equals(v.getType())) {
             throw new IOException("Error deserializing "+tag+".");
@@ -207,7 +206,7 @@ class XmlInputArchive implements InputArchive {
         return Utils.fromXMLString(v.getValue());
     }
     
-    public BytesWritable readBuffer(String tag) throws IOException {
+    public Buffer readBuffer(String tag) throws IOException {
         Value v = next();
         if (!"string".equals(v.getType())) {
             throw new IOException("Error deserializing "+tag+".");

src/java/org/apache/hadoop/record/XmlOutputArchive.java

@@ -24,8 +24,6 @@ import java.util.ArrayList;
 import java.io.PrintStream;
 import java.io.OutputStream;
 import java.util.Stack;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  *
@@ -191,7 +189,7 @@ class XmlOutputArchive implements OutputArchive {
         printEndEnvelope(tag);
     }
     
-    public void writeString(Text s, String tag) throws IOException {
+    public void writeString(String s, String tag) throws IOException {
         printBeginEnvelope(tag);
         stream.print("<string>");
         stream.print(Utils.toXMLString(s));
@@ -199,7 +197,7 @@ class XmlOutputArchive implements OutputArchive {
         printEndEnvelope(tag);
     }
     
-    public void writeBuffer(BytesWritable buf, String tag)
+    public void writeBuffer(Buffer buf, String tag)
     throws IOException {
         printBeginEnvelope(tag);
         stream.print("<string>");

src/java/org/apache/hadoop/record/compiler/CGenerator.java

@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record.compiler;
+
+import java.util.ArrayList;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * C Code generator front-end for Hadoop record I/O.
+ *
+ * @author Milind Bhandarkar
+ */
+class CGenerator extends CodeGenerator {
+  
+  CGenerator() {
+  }
+  
+  /**
+   * Generate C code. This method only creates the requested file(s)
+   * and spits-out file-level elements (such as include statements etc.)
+   * record-level code is generated by JRecord.
+   */
+  void genCode(String name, ArrayList<JFile> ilist,
+      ArrayList<JRecord> rlist, String destDir, ArrayList<String> options)
+      throws IOException {
+    name = new File(destDir, (new File(name)).getName()).getAbsolutePath();
+    FileWriter cc = new FileWriter(name+".c");
+    FileWriter hh = new FileWriter(name+".h");
+    
+    hh.write("#ifndef __"+name.toUpperCase().replace('.','_')+"__\n");
+    hh.write("#define __"+name.toUpperCase().replace('.','_')+"__\n");
+    hh.write("#include \"recordio.h\"\n");
+    for (Iterator<JFile> iter = ilist.iterator(); iter.hasNext();) {
+      hh.write("#include \""+iter.next().getName()+".h\"\n");
+    }
+    
+    cc.write("#include \""+name+".h\"\n");
+    
+    /*
+    for (Iterator<JRecord> iter = rlist.iterator(); iter.hasNext();) {
+      iter.next().genCppCode(hh, cc);
+    }
+    */
+    
+    hh.write("#endif //"+name.toUpperCase().replace('.','_')+"__\n");
+    
+    hh.close();
+    cc.close();
+  }
+}

src/java/org/apache/hadoop/record/compiler/CodeBuffer.java

@@ -0,0 +1,96 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.record.compiler;
+
+import java.util.ArrayList;
+
+/**
+ * A wrapper around StringBuffer that automatically does indentation
+ *
+ * @author Milind Bhandarkar
+ */
+public class CodeBuffer {
+  
+  static private ArrayList<Character> startMarkers = new ArrayList<Character>();
+  static private ArrayList<Character> endMarkers = new ArrayList<Character>();
+  
+  static {
+    addMarkers('{', '}');
+    addMarkers('(', ')');
+  }
+  
+  static void addMarkers(char ch1, char ch2) {
+    startMarkers.add(ch1);
+    endMarkers.add(ch2);
+  }
+  
+  private int level = 0;
+  private int numSpaces = 2;
+  private boolean firstChar = true;
+  private StringBuffer sb;
+  
+  /** Creates a new instance of CodeBuffer */
+  CodeBuffer() {
+    this(2, "");
+  }
+  
+  CodeBuffer(String s) {
+    this(2, s);
+  }
+  
+  CodeBuffer(int numSpaces, String s) {
+    sb = new StringBuffer();
+    this.numSpaces = numSpaces;
+    this.append(s);
+  }
+  
+  void append(String s) {
+    int length = s.length();
+    for (int idx = 0; idx < length; idx++) {
+      char ch = s.charAt(idx);
+      append(ch);
+    }
+  }
+  
+  void append(char ch) {
+    if (endMarkers.contains(ch)) {
+      level--;
+    }
+    if (firstChar) {
+      for (int idx = 0; idx < level; idx++) {
+        for (int num = 0; num < numSpaces; num++) {
+          rawAppend(' ');
+        }
+      }
+    }
+    rawAppend(ch);
+    firstChar = false;
+    if (startMarkers.contains(ch)) {
+      level++;
+    }
+    if (ch == '\n') {
+      firstChar = true;
+    }
+  }
+
+  private void rawAppend(char ch) {
+    sb.append(ch);
+  }
+  
+  public String toString() {
+    return sb.toString();
+  }
+}

src/java/org/apache/hadoop/record/compiler/CodeGenerator.java

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.record.compiler;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+/**
+ * CodeGenerator is a Factory and a base class for Hadoop Record I/O translators.
+ * Different translators register creation methods with this factory.
+ *
+ * @author Milind Bhandarkar
+ */
+abstract class CodeGenerator {
+  
+  private static HashMap<String, CodeGenerator> generators =
+      new HashMap<String, CodeGenerator>();
+  
+  static {
+    register("c", new CGenerator());
+    register("c++", new CppGenerator());
+    register("java", new JavaGenerator());
+  }
+  
+  static void register(String lang, CodeGenerator gen) {
+    generators.put(lang, gen);
+  }
+  
+  static CodeGenerator get(String lang) {
+    return generators.get(lang);
+  }
+  
+  abstract void genCode(String file,
+      ArrayList<JFile> inclFiles,
+      ArrayList<JRecord> records,
+      String destDir,
+      ArrayList<String> options) throws IOException;
+}

src/java/org/apache/hadoop/record/compiler/CppGenerator.java

@@ -29,50 +29,39 @@ import java.util.Iterator;
  *
  * @author Milind Bhandarkar
  */
-class CppGenerator {
-    private String mName;
-    private ArrayList mInclFiles;
-    private ArrayList mRecList;
+class CppGenerator extends CodeGenerator {
+  
+  CppGenerator() {
+  }
+  
+  /**
+   * Generate C++ code. This method only creates the requested file(s)
+   * and spits-out file-level elements (such as include statements etc.)
+   * record-level code is generated by JRecord.
+   */
+  void genCode(String name, ArrayList<JFile> ilist,
+      ArrayList<JRecord> rlist, String destDir, ArrayList<String> options)
+      throws IOException {
+    name = new File(destDir, (new File(name)).getName()).getAbsolutePath();
+    FileWriter cc = new FileWriter(name+".cc");
+    FileWriter hh = new FileWriter(name+".hh");
     
-    /** Creates a new instance of CppGenerator
-     *
-     * @param name possibly full pathname to the file
-     * @param ilist included files (as JFile)
-     * @param rlist List of records defined within this file
-     * @param destDir Output directory
-     */
-    CppGenerator(String name, ArrayList ilist, ArrayList rlist, String destDir) {
-        mName = new File(destDir, (new File(name)).getName()).getAbsolutePath();
-        mInclFiles = ilist;
-        mRecList = rlist;
+    hh.write("#ifndef __"+name.toUpperCase().replace('.','_')+"__\n");
+    hh.write("#define __"+name.toUpperCase().replace('.','_')+"__\n");
+    hh.write("#include \"recordio.hh\"\n");
+    for (Iterator<JFile> iter = ilist.iterator(); iter.hasNext();) {
+      hh.write("#include \""+iter.next().getName()+".hh\"\n");
     }
     
-    /**
-     * Generate C++ code. This method only creates the requested file(s)
-     * and spits-out file-level elements (such as include statements etc.)
-     * record-level code is generated by JRecord.
-     */
-    void genCode() throws IOException {
-        FileWriter cc = new FileWriter(mName+".cc");
-        FileWriter hh = new FileWriter(mName+".hh");
-        hh.write("#ifndef __"+mName.toUpperCase().replace('.','_')+"__\n");
-        hh.write("#define __"+mName.toUpperCase().replace('.','_')+"__\n");
-        
-        hh.write("#include \"recordio.hh\"\n");
-        for (Iterator i = mInclFiles.iterator(); i.hasNext();) {
-            JFile f = (JFile) i.next();
-            hh.write("#include \""+f.getName()+".hh\"\n");
-        }
-        cc.write("#include \""+mName+".hh\"\n");
-        
-        for (Iterator i = mRecList.iterator(); i.hasNext();) {
-            JRecord jr = (JRecord) i.next();
-            jr.genCppCode(hh, cc);
-        }
-        
-        hh.write("#endif //"+mName.toUpperCase().replace('.','_')+"__\n");
-        
-        hh.close();
-        cc.close();
+    cc.write("#include \""+name+".hh\"\n");
+    
+    for (Iterator<JRecord> iter = rlist.iterator(); iter.hasNext();) {
+      iter.next().genCppCode(hh, cc, options);
     }
+    
+    hh.write("#endif //"+name.toUpperCase().replace('.','_')+"__\n");
+    
+    hh.close();
+    cc.close();
+  }
 }

src/java/org/apache/hadoop/record/compiler/JBoolean.java

@@ -18,63 +18,63 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JType.CType;
+import org.apache.hadoop.record.compiler.JType.CppType;
+
 /**
  *
  * @author Milind Bhandarkar
  */
 public class JBoolean extends JType {
+  
+  class JavaBoolean extends JType.JavaType {
     
-    /** Creates a new instance of JBoolean */
-    public JBoolean() {
-        super("bool", "boolean", "Bool", "Boolean", "toBoolean");
-    }
-    
-    public String getSignature() {
-        return "z";
-    }
-    
-    String getJavaCompareToWrapper(String fname, String other) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("          boolean ee1 = ("+fname+".toBoolean()\n");
-      sb.append("          boolean ee2 = ("+other+".toBoolean()\n");
-      sb.append("          ret = (ee1 == ee2)? 0 :(ee1?-1:1);\n");
-      sb.append("        }\n");
-      return sb.toString();
+    JavaBoolean() {
+      super("boolean", "Bool", "Boolean");
     }
     
-    public String genJavaCompareTo(String fname, String other) {
-        return "    ret = ("+fname+" == "+other+")? 0 : ("+fname+"?1:-1);\n";
+    void genCompareTo(CodeBuffer cb, String fname, String other) {
+      cb.append("ret = ("+fname+" == "+other+")? 0 : ("+fname+"?1:-1);\n");
     }
     
-    public String genJavaHashCode(String fname) {
-        return "     ret = ("+fname+")?0:1;\n";
+    void genHashCode(CodeBuffer cb, String fname) {
+      cb.append("ret = ("+fname+")?0:1;\n");
     }
     
     // In Binary format, boolean is written as byte. true = 1, false = 0
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if ("+l+"<1) {\n");
-      sb.append("             throw new IOException(\"Boolean is exactly 1 byte. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           "+s+"++; "+l+"--;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("if ("+l+"<1) {\n");
+      cb.append("throw new java.io.IOException(\"Boolean is exactly 1 byte."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append(s+"++; "+l+"--;\n");
+      cb.append("}\n");
     }
     
     // In Binary format, boolean is written as byte. true = 1, false = 0
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if (l1<1 || l2<1) {\n");
-      sb.append("             throw new IOException(\"Boolean is exactly 1 byte. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           if (b1[s1] != b2[s2]) {\n");
-      sb.append("             return (b1[s1]<b2[s2])? -1 : 0;\n");
-      sb.append("           }\n");
-      sb.append("           s1++; s2++; l1--; l2--;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("if (l1<1 || l2<1) {\n");
+      cb.append("throw new java.io.IOException(\"Boolean is exactly 1 byte."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append("if (b1[s1] != b2[s2]) {\n");
+      cb.append("return (b1[s1]<b2[s2])? -1 : 0;\n");
+      cb.append("}\n");
+      cb.append("s1++; s2++; l1--; l2--;\n");
+      cb.append("}\n");
     }
+  }
+  
+  /** Creates a new instance of JBoolean */
+  public JBoolean() {
+    setJavaType(new JavaBoolean());
+    setCppType(new CppType("bool"));
+    setCType(new CType());
+  }
+  
+  String getSignature() {
+    return "z";
+  }
 }

src/java/org/apache/hadoop/record/compiler/JBuffer.java

@@ -18,88 +18,83 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JCompType.CCompType;
+
 /**
+ * Code generator for "buffer" type.
  *
  * @author Milind Bhandarkar
  */
 public class JBuffer extends JCompType {
+  
+  class JavaBuffer extends JavaCompType {
     
-    /** Creates a new instance of JBuffer */
-    public JBuffer() {
-        super(" ::std::string", "BytesWritable", "Buffer", "BytesWritable");
-    }
-    
-    public String genCppGetSet(String fname, int fIdx) {
-        String cgetFunc = "  virtual const "+getCppType()+"& get"+fname+"() const {\n";
-        cgetFunc += "    return m"+fname+";\n";
-        cgetFunc += "  }\n";
-        String getFunc = "  virtual "+getCppType()+"& get"+fname+"() {\n";
-        getFunc += "    return m"+fname+";\n";
-        getFunc += "  }\n";
-        return cgetFunc + getFunc;
-    }
-    
-    public String getSignature() {
-        return "B";
+    JavaBuffer() {
+      super("org.apache.hadoop.record.Buffer", "Buffer", "org.apache.hadoop.record.Buffer");
     }
     
-    public String genJavaReadWrapper(String fname, String tag, boolean decl) {
-        String ret = "";
-        if (decl) {
-            ret = "    BytesWritable "+fname+";\n";
-        }
-        return ret + "        "+fname+"=a_.readBuffer(\""+tag+"\");\n";
+    void genWriteMethod(CodeBuffer cb, String fname, String tag) {
+      cb.append("a_.writeBuffer("+fname+",\""+tag+"\");\n");
     }
     
-    public String genJavaWriteWrapper(String fname, String tag) {
-        return "        a_.writeBuffer("+fname+",\""+tag+"\");\n";
+    void genCompareTo(CodeBuffer cb, String fname, String other) {
+      cb.append("ret = "+fname+".compareTo("+other+");\n");
     }
     
-    public String genJavaCompareTo(String fname, String other) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("    {\n");
-      sb.append("      byte[] my = "+fname+".get();\n");
-      sb.append("      byte[] ur = "+other+".get();\n");
-      sb.append("      ret = WritableComparator.compareBytes(my,0,"+
-          fname+".getSize(),ur,0,"+other+".getSize());\n");
-      sb.append("    }\n");
-      return sb.toString();
+    void genEquals(CodeBuffer cb, String fname, String peer) {
+      cb.append("ret = "+fname+".equals("+peer+");\n");
     }
     
-    public String genJavaCompareToWrapper(String fname, String other) {
-      return "    "+genJavaCompareTo(fname, other);
+    void genHashCode(CodeBuffer cb, String fname) {
+      cb.append("ret = "+fname+".hashCode();\n");
     }
     
-    public String genJavaEquals(String fname, String peer) {
-        return "    ret = "+fname+".equals("+peer+");\n";
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("int i = org.apache.hadoop.record.Utils.readVInt("+
+          b+", "+s+");\n");
+      cb.append("int z = org.apache.hadoop.record.Utils.getVIntSize(i);\n");
+      cb.append(s+" += z+i; "+l+" -= (z+i);\n");
+      cb.append("}\n");
     }
     
-    public String genJavaHashCode(String fname) {
-        return "    ret = "+fname+".hashCode();\n";
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("int i1 = org.apache.hadoop.record.Utils.readVInt(b1, s1);\n");
+      cb.append("int i2 = org.apache.hadoop.record.Utils.readVInt(b2, s2);\n");
+      cb.append("int z1 = org.apache.hadoop.record.Utils.getVIntSize(i1);\n");
+      cb.append("int z2 = org.apache.hadoop.record.Utils.getVIntSize(i2);\n");
+      cb.append("s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
+      cb.append("int r1 = org.apache.hadoop.record.Utils.compareBytes(b1,s1,l1,b2,s2,l2);\n");
+      cb.append("if (r1 != 0) { return (r1<0)?-1:0; }\n");
+      cb.append("s1+=i1; s2+=i2; l1-=i1; l1-=i2;\n");
+      cb.append("}\n");
     }
+  }
+  
+  class CppBuffer extends CppCompType {
     
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           int i = WritableComparator.readVInt("+b+", "+s+");\n");
-      sb.append("           int z = WritableUtils.getVIntSize(i);\n");
-      sb.append("           "+s+" += z+i; "+l+" -= (z+i);\n");
-      sb.append("        }\n");
-      return sb.toString();
+    CppBuffer() {
+      super(" ::std::string");
     }
     
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           int i1 = WritableComparator.readVInt(b1, s1);\n");
-      sb.append("           int i2 = WritableComparator.readVInt(b2, s2);\n");
-      sb.append("           int z1 = WritableUtils.getVIntSize(i1);\n");
-      sb.append("           int z2 = WritableUtils.getVIntSize(i2);\n");
-      sb.append("           s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
-      sb.append("           int r1 = WritableComparator.compareBytes(b1,s1,l1,b2,s2,l2);\n");
-      sb.append("           if (r1 != 0) { return (r1<0)?-1:0; }\n");
-      sb.append("           s1+=i1; s2+=i2; l1-=i1; l1-=i2;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genGetSet(CodeBuffer cb, String fname) {
+      cb.append("virtual const "+getType()+"& get"+toCamelCase(fname)+"() const {\n");
+      cb.append("return "+fname+";\n");
+      cb.append("}\n");
+      cb.append("virtual "+getType()+"& get"+toCamelCase(fname)+"() {\n");
+      cb.append("return "+fname+";\n");
+      cb.append("}\n");
     }
+  }
+  /** Creates a new instance of JBuffer */
+  public JBuffer() {
+    setJavaType(new JavaBuffer());
+    setCppType(new CppBuffer());
+    setCType(new CCompType());
+  }
+  
+  String getSignature() {
+    return "B";
+  }
 }

src/java/org/apache/hadoop/record/compiler/JByte.java

@@ -18,43 +18,52 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JType.CType;
+import org.apache.hadoop.record.compiler.JType.CppType;
+
 /**
- *
+ * Code generator for "byte" type.
  * @author Milind Bhandarkar
  */
 public class JByte extends JType {
+  
+  class JavaByte extends JavaType {
     
-    /** Creates a new instance of JByte */
-    public JByte() {
-        super("int8_t", "byte", "Byte", "Byte", "toByte");
-    }
-    
-    public String getSignature() {
-        return "b";
+    JavaByte() {
+      super("byte", "Byte", "Byte");
     }
     
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if ("+l+"<1) {\n");
-      sb.append("             throw new IOException(\"Byte is exactly 1 byte. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           "+s+"++; "+l+"--;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("if ("+l+"<1) {\n");
+      cb.append("throw new java.io.IOException(\"Byte is exactly 1 byte."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append(s+"++; "+l+"--;\n");
+      cb.append("}\n");
     }
     
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if (l1<1 || l2<1) {\n");
-      sb.append("             throw new IOException(\"Byte is exactly 1 byte. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           if (b1[s1] != b2[s2]) {\n");
-      sb.append("             return (b1[s1]<b2[s2])?-1:0;\n");
-      sb.append("           }\n");
-      sb.append("           s1++; s2++; l1--; l2--;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("if (l1<1 || l2<1) {\n");
+      cb.append("throw new java.io.IOException(\"Byte is exactly 1 byte."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append("if (b1[s1] != b2[s2]) {\n");
+      cb.append("return (b1[s1]<b2[s2])?-1:0;\n");
+      cb.append("}\n");
+      cb.append("s1++; s2++; l1--; l2--;\n");
+      cb.append("}\n");
     }
+  }
+  
+  public JByte() {
+    setJavaType(new JavaByte());
+    setCppType(new CppType("int8_t"));
+    setCType(new CType());
+  }
+  
+  String getSignature() {
+    return "b";
+  }
 }

src/java/org/apache/hadoop/record/compiler/JCompType.java

@@ -25,35 +25,47 @@ package org.apache.hadoop.record.compiler;
  * @author Milind Bhandarkar
  */
 abstract class JCompType extends JType {
+  
+  abstract class JavaCompType extends JavaType {
     
-    /** Creates a new instance of JCompType */
-    JCompType(String cppType, String javaType, String suffix, String wrapper) {
-        super(cppType, javaType, suffix, wrapper, null);
+    JavaCompType(String type, String suffix, String wrapper) {
+      super(type, suffix, wrapper);
     }
     
-    String genCppGetSet(String fname, int fIdx) {
-        String cgetFunc = "  virtual const "+getCppType()+"& get"+fname+"() const {\n";
-        cgetFunc += "    return m"+fname+";\n";
-        cgetFunc += "  }\n";
-        String getFunc = "  virtual "+getCppType()+"& get"+fname+"() {\n";
-        getFunc += "    return m"+fname+";\n";
-        getFunc += "  }\n";
-        return cgetFunc + getFunc;
+    void genCompareTo(CodeBuffer cb, String fname, String other) {
+      cb.append("ret = "+fname+".compareTo("+other+");\n");
     }
     
-    String genJavaCompareTo(String fname, String other) {
-        return "    ret = "+fname+".compareTo("+other+");\n";
+    void genEquals(CodeBuffer cb, String fname, String peer) {
+      cb.append("ret = "+fname+".equals("+peer+");\n");
     }
     
-    String genJavaCompareToWrapper(String fname, String other) {
-        return "    "+genJavaCompareTo(fname, other);
+    void genHashCode(CodeBuffer cb, String fname) {
+      cb.append("ret = "+fname+".hashCode();\n");
     }
     
-    String genJavaEquals(String fname, String peer) {
-        return "    ret = "+fname+".equals("+peer+");\n";
+    void genClone(CodeBuffer cb, String fname) {
+      cb.append("other."+fname+" = ("+getType()+") this."+fname+".clone();\n");
     }
+  }
+  
+  class CppCompType extends CppType {
     
-    String genJavaHashCode(String fname) {
-        return "    ret = "+fname+".hashCode();\n";
+    CppCompType(String type) {
+      super(type);
     }
+    
+    void genGetSet(CodeBuffer cb, String fname) {
+      cb.append("virtual const "+getType()+"& get"+toCamelCase(fname)+"() const {\n");
+      cb.append("return "+fname+";\n");
+      cb.append("}\n");
+      cb.append("virtual "+getType()+"& get"+toCamelCase(fname)+"() {\n");
+      cb.append("return "+fname+";\n");
+      cb.append("}\n");
+    }
+  }
+  
+  class CCompType extends CType {
+    
+  }
 }

src/java/org/apache/hadoop/record/compiler/JDouble.java

@@ -18,50 +18,59 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JType.CType;
+import org.apache.hadoop.record.compiler.JType.CppType;
+
 /**
  *
  * @author Milind Bhandarkar
  */
 public class JDouble extends JType {
+  
+  class JavaDouble extends JavaType {
     
-    /** Creates a new instance of JDouble */
-    public JDouble() {
-        super("double", "double", "Double", "Double", "toDouble");
-    }
-    
-    public String getSignature() {
-        return "d";
+    JavaDouble() {
+      super("double", "Double", "Double");
     }
     
-    public String genJavaHashCode(String fname) {
-        String tmp = "Double.doubleToLongBits("+fname+")";
-        return "    ret = (int)("+tmp+"^("+tmp+">>>32));\n";
+    void genHashCode(CodeBuffer cb, String fname) {
+      String tmp = "Double.doubleToLongBits("+fname+")";
+      cb.append("ret = (int)("+tmp+"^("+tmp+">>>32));\n");
     }
     
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if ("+l+"<8) {\n");
-      sb.append("             throw new IOException(\"Double is exactly 8 bytes. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           "+s+"+=8; "+l+"-=8;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("if ("+l+"<8) {\n");
+      cb.append("throw new java.io.IOException(\"Double is exactly 8 bytes."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append(s+"+=8; "+l+"-=8;\n");
+      cb.append("}\n");
     }
     
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if (l1<8 || l2<8) {\n");
-      sb.append("             throw new IOException(\"Double is exactly 8 bytes. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           double d1 = WritableComparator.readDouble(b1, s1);\n");
-      sb.append("           double d2 = WritableComparator.readDouble(b2, s2);\n");
-      sb.append("           if (d1 != d2) {\n");
-      sb.append("             return ((d1-d2) < 0) ? -1 : 0;\n");
-      sb.append("           }\n");
-      sb.append("           s1+=8; s2+=8; l1-=8; l2-=8;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("if (l1<8 || l2<8) {\n");
+      cb.append("throw new java.io.IOException(\"Double is exactly 8 bytes."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append("double d1 = org.apache.hadoop.record.Utils.readDouble(b1, s1);\n");
+      cb.append("double d2 = org.apache.hadoop.record.Utils.readDouble(b2, s2);\n");
+      cb.append("if (d1 != d2) {\n");
+      cb.append("return ((d1-d2) < 0) ? -1 : 0;\n");
+      cb.append("}\n");
+      cb.append("s1+=8; s2+=8; l1-=8; l2-=8;\n");
+      cb.append("}\n");
     }
+  }
+  /** Creates a new instance of JDouble */
+  public JDouble() {
+    setJavaType(new JavaDouble());
+    setCppType(new CppType("double"));
+    setCType(new CType());
+  }
+  
+  String getSignature() {
+    return "d";
+  }
 }

src/java/org/apache/hadoop/record/compiler/JField.java

@@ -19,85 +19,28 @@
 package org.apache.hadoop.record.compiler;
 
 /**
+ * A thin wrappper around record field.
  *
  * @author Milind Bhandarkar
  */
-public class JField {
-    private JType mType;
-    private String mName;
-    /**
-     * Creates a new instance of JField
-     */
-    public JField(JType type, String name) {
-        mType = type;
-        mName = name;
-    }
-    
-    public String getSignature() {
-        return mType.getSignature();
-    }
-    
-    public String genCppDecl() {
-        return mType.genCppDecl(mName);
-    }
-    
-    public String genJavaDecl() {
-        return mType.genJavaDecl(mName);
-    }
-    
-    public String genJavaConstructorParam(int fIdx) {
-        return mType.genJavaConstructorParam(fIdx);
-    }
-    
-    public String getName() {
-        return "m"+mName;
-    }
-    
-    public String getTag() {
-        return mName;
-    }
-    
-    public JType getType() {
-        return mType;
-    }
-    
-    public String genCppGetSet(int fIdx) {
-        return mType.genCppGetSet(mName, fIdx);
-    }
-    
-    public String genJavaGetSet(int fIdx) {
-        return mType.genJavaGetSet(mName, fIdx);
-    }
-    
-    public String genJavaWriteMethodName() {
-        return mType.genJavaWriteMethod(getName(), getTag());
-    }
-    
-    public String genJavaReadMethodName() {
-        return mType.genJavaReadMethod(getName(), getTag());
-    }
-    
-    public String genJavaCompareTo() {
-        return mType.genJavaCompareTo(getName(), "peer."+getName());
-    }
-    
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      return mType.genJavaSlurpBytes(b,s,l);
-    }
-    
-    public String genJavaCompareBytes() {
-      return mType.genJavaCompareBytes();
-    }
-    
-    public String genJavaEquals() {
-        return mType.genJavaEquals(getName(), "peer."+getName());
-    }
-    
-    public String genJavaHashCode() {
-        return mType.genJavaHashCode(getName());
-    }
-
-    public String genJavaConstructorSet(int fIdx) {
-        return mType.genJavaConstructorSet(mName, fIdx);
-    }
+public class JField<T> {
+  
+  private String name;
+  private T type;
+  
+  /**
+   * Creates a new instance of JField
+   */
+  public JField(String name, T type) {
+    this.type = type;
+    this.name = name;
+  }
+  
+  String getName() {
+    return name;
+  }
+  
+  T getType() {
+    return type;
+  }
 }

src/java/org/apache/hadoop/record/compiler/JFile.java

@@ -29,10 +29,12 @@ import java.util.ArrayList;
  * @author Milind Bhandarkar
  */
 public class JFile {
-    
+    /** Possibly full name of the file */
     private String mName;
-    private ArrayList mInclFiles;
-    private ArrayList mRecords;
+    /** Ordered list of included files */
+    private ArrayList<JFile> mInclFiles;
+    /** Ordered list of records declared in this file */
+    private ArrayList<JRecord> mRecords;
     
     /** Creates a new instance of JFile
      *
@@ -40,7 +42,8 @@ public class JFile {
      * @param inclFiles included files (as JFile)
      * @param recList List of records defined within this file
      */
-    public JFile(String name, ArrayList inclFiles, ArrayList recList) {
+    public JFile(String name, ArrayList<JFile> inclFiles,
+        ArrayList<JRecord> recList) {
         mName = name;
         mInclFiles = inclFiles;
         mRecords = recList;
@@ -55,17 +58,15 @@ public class JFile {
     /** Generate record code in given language. Language should be all
      *  lowercase.
      */
-    public int genCode(String language, String destDir) throws IOException {
-        if ("c++".equals(language)) {
-            CppGenerator gen = new CppGenerator(mName, mInclFiles, mRecords, destDir);
-            gen.genCode();
-        } else if ("java".equals(language)) {
-            JavaGenerator gen = new JavaGenerator(mName, mInclFiles, mRecords, destDir);
-            gen.genCode();
-        } else {
-            System.err.println("Cannnot recognize language:"+language);
-            return 1;
-        }
-        return 0;
+    public int genCode(String language, String destDir, ArrayList<String> options)
+    throws IOException {
+      CodeGenerator gen = CodeGenerator.get(language);
+      if (gen != null) {
+        gen.genCode(mName, mInclFiles, mRecords, destDir, options);
+      } else {
+        System.err.println("Cannnot recognize language:"+language);
+        return 1;
+      }
+      return 0;
     }
 }

src/java/org/apache/hadoop/record/compiler/JFloat.java

@@ -18,49 +18,58 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JType.CType;
+import org.apache.hadoop.record.compiler.JType.CppType;
+
 /**
  *
  * @author Milind Bhandarkar
  */
 public class JFloat extends JType {
+  
+  class JavaFloat extends JavaType {
     
-    /** Creates a new instance of JFloat */
-    public JFloat() {
-        super("float", "float", "Float", "Float", "toFloat");
-    }
-    
-    public String getSignature() {
-        return "f";
+    JavaFloat() {
+      super("float", "Float", "Float");
     }
     
-    public String genJavaHashCode(String fname) {
-        return "    ret = Float.floatToIntBits("+fname+");\n";
+    void genHashCode(CodeBuffer cb, String fname) {
+      cb.append("ret = Float.floatToIntBits("+fname+");\n");
     }
     
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if ("+l+"<4) {\n");
-      sb.append("             throw new IOException(\"Float is exactly 4 bytes. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           "+s+"+=4; "+l+"-=4;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("if ("+l+"<4) {\n");
+      cb.append("throw new java.io.IOException(\"Float is exactly 4 bytes."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append(s+"+=4; "+l+"-=4;\n");
+      cb.append("}\n");
     }
     
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if (l1<4 || l2<4) {\n");
-      sb.append("             throw new IOException(\"Float is exactly 4 bytes. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           float f1 = WritableComparator.readFloat(b1, s1);\n");
-      sb.append("           float f2 = WritableComparator.readFloat(b2, s2);\n");
-      sb.append("           if (f1 != f2) {\n");
-      sb.append("             return ((f1-f2) < 0) ? -1 : 0;\n");
-      sb.append("           }\n");
-      sb.append("           s1+=4; s2+=4; l1-=4; l2-=4;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("if (l1<4 || l2<4) {\n");
+      cb.append("throw new java.io.IOException(\"Float is exactly 4 bytes."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append("float f1 = org.apache.hadoop.record.Utils.readFloat(b1, s1);\n");
+      cb.append("float f2 = org.apache.hadoop.record.Utils.readFloat(b2, s2);\n");
+      cb.append("if (f1 != f2) {\n");
+      cb.append("return ((f1-f2) < 0) ? -1 : 0;\n");
+      cb.append("}\n");
+      cb.append("s1+=4; s2+=4; l1-=4; l2-=4;\n");
+      cb.append("}\n");
     }
+  }
+  /** Creates a new instance of JFloat */
+  public JFloat() {
+    setJavaType(new JavaFloat());
+    setCppType(new CppType("float"));
+    setCType(new CType());
+  }
+  
+  String getSignature() {
+    return "f";
+  }
 }

src/java/org/apache/hadoop/record/compiler/JInt.java

@@ -18,43 +18,50 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JType.CType;
+import org.apache.hadoop.record.compiler.JType.CppType;
+
 /**
- *
+ * Code generator for "int" type
  * @author Milind Bhandarkar
  */
 public class JInt extends JType {
+  
+  class JavaInt extends JavaType {
     
-    /** Creates a new instance of JInt */
-    public JInt() {
-        super("int32_t", "int", "Int", "Integer", "toInt");
-    }
-    
-    public String getSignature() {
-        return "i";
+    JavaInt() {
+      super("int", "Int", "Integer");
     }
     
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           int i = WritableComparator.readVInt("+b+", "+s+");\n");
-      sb.append("           int z = WritableUtils.getVIntSize(i);\n");
-      sb.append("           "+s+"+=z; "+l+"-=z;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("int i = org.apache.hadoop.record.Utils.readVInt("+b+", "+s+");\n");
+      cb.append("int z = org.apache.hadoop.record.Utils.getVIntSize(i);\n");
+      cb.append(s+"+=z; "+l+"-=z;\n");
+      cb.append("}\n");
     }
     
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           int i1 = WritableComparator.readVInt(b1, s1);\n");
-      sb.append("           int i2 = WritableComparator.readVInt(b2, s2);\n");
-      sb.append("           if (i1 != i2) {\n");
-      sb.append("             return ((i1-i2) < 0) ? -1 : 0;\n");
-      sb.append("           }\n");
-      sb.append("           int z1 = WritableUtils.getVIntSize(i1);\n");
-      sb.append("           int z2 = WritableUtils.getVIntSize(i2);\n");
-      sb.append("           s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("int i1 = org.apache.hadoop.record.Utils.readVInt(b1, s1);\n");
+      cb.append("int i2 = org.apache.hadoop.record.Utils.readVInt(b2, s2);\n");
+      cb.append("if (i1 != i2) {\n");
+      cb.append("return ((i1-i2) < 0) ? -1 : 0;\n");
+      cb.append("}\n");
+      cb.append("int z1 = org.apache.hadoop.record.Utils.getVIntSize(i1);\n");
+      cb.append("int z2 = org.apache.hadoop.record.Utils.getVIntSize(i2);\n");
+      cb.append("s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
+      cb.append("}\n");
     }
+  }
+  /** Creates a new instance of JInt */
+  public JInt() {
+    setJavaType(new JavaInt());
+    setCppType(new CppType("int32_t"));
+    setCType(new CType());
+  }
+  
+  String getSignature() {
+    return "i";
+  }
 }

src/java/org/apache/hadoop/record/compiler/JLong.java

@@ -18,47 +18,54 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JType.CType;
+import org.apache.hadoop.record.compiler.JType.CppType;
+
 /**
- *
+ * Code generator for "long" type
  * @author Milind Bhandarkar
  */
 public class JLong extends JType {
+  
+  class JavaLong extends JavaType {
     
-    /** Creates a new instance of JLong */
-    public JLong() {
-        super("int64_t", "long", "Long", "Long", "toLong");
-    }
-    
-    public String getSignature() {
-        return "l";
+    JavaLong() {
+      super("long", "Long", "Long");
     }
     
-    public String genJavaHashCode(String fname) {
-        return "    ret = (int) ("+fname+"^("+fname+">>>32));\n";
+    void genHashCode(CodeBuffer cb, String fname) {
+      cb.append("ret = (int) ("+fname+"^("+fname+">>>32));\n");
     }
     
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           long i = WritableComparator.readVLong("+b+", "+s+");\n");
-      sb.append("           int z = WritableUtils.getVIntSize(i);\n");
-      sb.append("           "+s+"+=z; "+l+"-=z;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("long i = org.apache.hadoop.record.Utils.readVLong("+b+", "+s+");\n");
+      cb.append("int z = org.apache.hadoop.record.Utils.getVIntSize(i);\n");
+      cb.append(s+"+=z; "+l+"-=z;\n");
+      cb.append("}\n");
     }
     
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           long i1 = WritableComparator.readVLong(b1, s1);\n");
-      sb.append("           long i2 = WritableComparator.readVLong(b2, s2);\n");
-      sb.append("           if (i1 != i2) {\n");
-      sb.append("             return ((i1-i2) < 0) ? -1 : 0;\n");
-      sb.append("           }\n");
-      sb.append("           int z1 = WritableUtils.getVIntSize(i1);\n");
-      sb.append("           int z2 = WritableUtils.getVIntSize(i2);\n");
-      sb.append("           s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("long i1 = org.apache.hadoop.record.Utils.readVLong(b1, s1);\n");
+      cb.append("long i2 = org.apache.hadoop.record.Utils.readVLong(b2, s2);\n");
+      cb.append("if (i1 != i2) {\n");
+      cb.append("return ((i1-i2) < 0) ? -1 : 0;\n");
+      cb.append("}\n");
+      cb.append("int z1 = org.apache.hadoop.record.Utils.getVIntSize(i1);\n");
+      cb.append("int z2 = org.apache.hadoop.record.Utils.getVIntSize(i2);\n");
+      cb.append("s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
+      cb.append("}\n");
     }
+  }
+  /** Creates a new instance of JLong */
+  public JLong() {
+    setJavaType(new JavaLong());
+    setCppType(new CppType("int64_t"));
+    setCType(new CType());
+  }
+  
+  String getSignature() {
+    return "l";
+  }
 }

src/java/org/apache/hadoop/record/compiler/JMap.java

@@ -23,149 +23,152 @@ package org.apache.hadoop.record.compiler;
  * @author Milind Bhandarkar
  */
 public class JMap extends JCompType {
-   
-    static private int level = 0;
+  
+  static private int level = 0;
+  
+  static private String getLevel() { return Integer.toString(level); }
+  
+  static private void incrLevel() { level++; }
+  
+  static private void decrLevel() { level--; }
+  
+  static private String getId(String id) { return id+getLevel(); }
+  
+  private JType keyType;
+  private JType valueType;
+  
+  class JavaMap extends JavaCompType {
     
-    static private String getLevel() { return Integer.toString(level); }
+    JType.JavaType key;
+    JType.JavaType value;
     
-    static private void incrLevel() { level++; }
-    
-    static private void decrLevel() { level--; }
-    
-    static private String getId(String id) { return id+getLevel(); }
-    
-    private JType mKey;
-    private JType mValue;
-    
-    /** Creates a new instance of JMap */
-    public JMap(JType t1, JType t2) {
-        super(" ::std::map<"+t1.getCppType()+","+t2.getCppType()+">",
-                "java.util.TreeMap", "Map", "java.util.TreeMap");
-        mKey = t1;
-        mValue = t2;
+    JavaMap(JType.JavaType key, JType.JavaType value) {
+      super("java.util.TreeMap<"+key.getWrapperType()+","+value.getWrapperType()+">",
+          "Map",
+          "java.util.TreeMap<"+key.getWrapperType()+","+value.getWrapperType()+">");
+      this.key = key;
+      this.value = value;
     }
     
-    public String getSignature() {
-        return "{" + mKey.getSignature() + mValue.getSignature() +"}";
-    }
-    
-    public String genJavaCompareTo(String fname, String other) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("    {\n");
-      sb.append("      java.util.Set "+getId("set1")+" = "+fname+".keySet();\n");
-      sb.append("      java.util.Set "+getId("set2")+" = "+other+".keySet();\n");
-      sb.append("      java.util.Iterator "+getId("miter1")+" = "+
+    void genCompareTo(CodeBuffer cb, String fname, String other) {
+      String setType = "java.util.Set<"+key.getWrapperType()+"> ";
+      String iterType = "java.util.Iterator<"+key.getWrapperType()+"> ";
+      cb.append("{\n");
+      cb.append(setType+getId("set1")+" = "+fname+".keySet();\n");
+      cb.append(setType+getId("set2")+" = "+other+".keySet();\n");
+      cb.append(iterType+getId("miter1")+" = "+
           getId("set1")+".iterator();\n");
-      sb.append("      java.util.Iterator "+getId("miter2")+" = "+
+      cb.append(iterType+getId("miter2")+" = "+
           getId("set2")+".iterator();\n");
-      sb.append("      for(; "+getId("miter1")+".hasNext() && "+
+      cb.append("for(; "+getId("miter1")+".hasNext() && "+
           getId("miter2")+".hasNext(); ) {\n");
-      sb.append("        "+mKey.getJavaWrapperType()+" "+getId("k1")+
-          " = ("+mKey.getJavaWrapperType()+") "+getId("miter1")+".next();\n");
-      sb.append("        "+mKey.getJavaWrapperType()+" "+getId("k2")+
-          " = ("+mKey.getJavaWrapperType()+") "+getId("miter2")+".next();\n");
-      sb.append(mKey.genJavaCompareToWrapper(getId("k1"), getId("k2")));
-      sb.append("         if (ret != 0) { return ret; }\n");
-      sb.append("      }\n");
-      sb.append("      ret = ("+getId("set1")+".size() - "+getId("set2")+".size());\n");
-      sb.append("    }\n");
-      return sb.toString();
-    }
-    
-    public String genJavaCompareToWrapper(String fname, String other) {
-      return genJavaCompareTo(fname, other);
+      cb.append(key.getType()+" "+getId("k1")+
+          " = "+getId("miter1")+".next();\n");
+      cb.append(key.getType()+" "+getId("k2")+
+          " = "+getId("miter2")+".next();\n");
+      key.genCompareTo(cb, getId("k1"), getId("k2"));
+      cb.append("if (ret != 0) { return ret; }\n");
+      cb.append("}\n");
+      cb.append("ret = ("+getId("set1")+".size() - "+getId("set2")+".size());\n");
+      cb.append("}\n");
     }
     
-    public String genJavaReadWrapper(String fname, String tag, boolean decl) {
-        StringBuffer ret = new StringBuffer("");
-        if (decl) {
-            ret.append("    java.util.TreeMap "+fname+";\n");
-        }
-        ret.append("    {\n");
-        incrLevel();
-        ret.append("      org.apache.hadoop.record.Index "+getId("midx")+" = a_.startMap(\""+tag+"\");\n");
-        ret.append("      "+fname+"=new java.util.TreeMap();\n");
-        ret.append("      for (; !"+getId("midx")+".done(); "+getId("midx")+".incr()) {\n");
-        ret.append(mKey.genJavaReadWrapper(getId("k"),getId("k"),true));
-        ret.append(mValue.genJavaReadWrapper(getId("v"),getId("v"),true));
-        ret.append("        "+fname+".put("+getId("k")+","+getId("v")+");\n");
-        ret.append("      }\n");
-        ret.append("    a_.endMap(\""+tag+"\");\n");
-        decrLevel();
-        ret.append("    }\n");
-        return ret.toString();
-    }
-    
-    public String genJavaReadMethod(String fname, String tag) {
-        return genJavaReadWrapper(fname, tag, false);
-    }
-    
-    public String genJavaWriteWrapper(String fname, String tag) {
-        StringBuffer ret = new StringBuffer("    {\n");
-        incrLevel();
-        ret.append("      a_.startMap("+fname+",\""+tag+"\");\n");
-        ret.append("      java.util.Set "+getId("es")+" = "+fname+".entrySet();\n");
-        ret.append("      for(java.util.Iterator "+getId("midx")+" = "+getId("es")+".iterator(); "+getId("midx")+".hasNext(); ) {\n");
-        ret.append("        java.util.Map.Entry "+getId("me")+" = (java.util.Map.Entry) "+getId("midx")+".next();\n");
-        ret.append("        "+mKey.getJavaWrapperType()+" "+getId("k")+" = ("+mKey.getJavaWrapperType()+") "+getId("me")+".getKey();\n");
-        ret.append("        "+mValue.getJavaWrapperType()+" "+getId("v")+" = ("+mValue.getJavaWrapperType()+") "+getId("me")+".getValue();\n");
-        ret.append(mKey.genJavaWriteWrapper(getId("k"),getId("k")));
-        ret.append(mValue.genJavaWriteWrapper(getId("v"),getId("v")));
-        ret.append("      }\n");
-        ret.append("      a_.endMap("+fname+",\""+tag+"\");\n");
-        ret.append("    }\n");
-        decrLevel();
-        return ret.toString();
+    void genReadMethod(CodeBuffer cb, String fname, String tag, boolean decl) {
+      if (decl) {
+        cb.append(getType()+" "+fname+";\n");
+      }
+      cb.append("{\n");
+      incrLevel();
+      cb.append("org.apache.hadoop.record.Index "+getId("midx")+" = a_.startMap(\""+tag+"\");\n");
+      cb.append(fname+"=new "+getType()+"();\n");
+      cb.append("for (; !"+getId("midx")+".done(); "+getId("midx")+".incr()) {\n");
+      key.genReadMethod(cb, getId("k"),getId("k"),true);
+      value.genReadMethod(cb, getId("v"),getId("v"),true);
+      cb.append(fname+".put("+getId("k")+","+getId("v")+");\n");
+      cb.append("}\n");
+      cb.append("a_.endMap(\""+tag+"\");\n");
+      decrLevel();
+      cb.append("}\n");
     }
     
-    public String genJavaWriteMethod(String fname, String tag) {
-        return genJavaWriteWrapper(fname, tag);
+    void genWriteMethod(CodeBuffer cb, String fname, String tag) {
+      String setType = "java.util.Set<java.util.Map.Entry<"+
+          key.getWrapperType()+","+value.getWrapperType()+">> ";
+      String entryType = "java.util.Map.Entry<"+
+          key.getWrapperType()+","+value.getWrapperType()+"> ";
+      String iterType = "java.util.Iterator<java.util.Map.Entry<"+
+          key.getWrapperType()+","+value.getWrapperType()+">> ";
+      cb.append("{\n");
+      incrLevel();
+      cb.append("a_.startMap("+fname+",\""+tag+"\");\n");
+      cb.append(setType+getId("es")+" = "+fname+".entrySet();\n");
+      cb.append("for("+iterType+getId("midx")+" = "+getId("es")+".iterator(); "+getId("midx")+".hasNext(); ) {\n");
+      cb.append(entryType+getId("me")+" = "+getId("midx")+".next();\n");
+      cb.append(key.getType()+" "+getId("k")+" = "+getId("me")+".getKey();\n");
+      cb.append(value.getType()+" "+getId("v")+" = "+getId("me")+".getValue();\n");
+      key.genWriteMethod(cb, getId("k"),getId("k"));
+      value.genWriteMethod(cb, getId("v"),getId("v"));
+      cb.append("}\n");
+      cb.append("a_.endMap("+fname+",\""+tag+"\");\n");
+      cb.append("}\n");
+      decrLevel();
     }
     
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
       incrLevel();
-      sb.append("           int "+getId("mi")+
-          " = WritableComparator.readVInt("+b+", "+s+");\n");
-      sb.append("           int "+getId("mz")+
-          " = WritableUtils.getVIntSize("+getId("mi")+");\n");
-      sb.append("           "+s+"+="+getId("mz")+"; "+l+"-="+getId("mz")+";\n");
-      sb.append("           for (int "+getId("midx")+" = 0; "+getId("midx")+
+      cb.append("int "+getId("mi")+
+          " = org.apache.hadoop.record.Utils.readVInt("+b+", "+s+");\n");
+      cb.append("int "+getId("mz")+
+          " = org.apache.hadoop.record.Utils.getVIntSize("+getId("mi")+");\n");
+      cb.append(s+"+="+getId("mz")+"; "+l+"-="+getId("mz")+";\n");
+      cb.append("for (int "+getId("midx")+" = 0; "+getId("midx")+
           " < "+getId("mi")+"; "+getId("midx")+"++) {");
-      sb.append(mKey.genJavaSlurpBytes(b,s,l));
-      sb.append(mValue.genJavaSlurpBytes(b,s,l));
-      sb.append("           }\n");
+      key.genSlurpBytes(cb, b,s,l);
+      value.genSlurpBytes(cb, b,s,l);
+      cb.append("}\n");
       decrLevel();
-      sb.append("        }\n");
-      return sb.toString();
+      cb.append("}\n");
     }
     
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
       incrLevel();
-      sb.append("           int "+getId("mi1")+
-          " = WritableComparator.readVInt(b1, s1);\n");
-      sb.append("           int "+getId("mi2")+
-          " = WritableComparator.readVInt(b2, s2);\n");
-      sb.append("           int "+getId("mz1")+
-          " = WritableUtils.getVIntSize("+getId("mi1")+");\n");
-      sb.append("           int "+getId("mz2")+
-          " = WritableUtils.getVIntSize("+getId("mi2")+");\n");
-      sb.append("           s1+="+getId("mz1")+"; s2+="+getId("mz2")+
+      cb.append("int "+getId("mi1")+
+          " = org.apache.hadoop.record.Utils.readVInt(b1, s1);\n");
+      cb.append("int "+getId("mi2")+
+          " = org.apache.hadoop.record.Utils.readVInt(b2, s2);\n");
+      cb.append("int "+getId("mz1")+
+          " = org.apache.hadoop.record.Utils.getVIntSize("+getId("mi1")+");\n");
+      cb.append("int "+getId("mz2")+
+          " = org.apache.hadoop.record.Utils.getVIntSize("+getId("mi2")+");\n");
+      cb.append("s1+="+getId("mz1")+"; s2+="+getId("mz2")+
           "; l1-="+getId("mz1")+"; l2-="+getId("mz2")+";\n");
-      sb.append("           for (int "+getId("midx")+" = 0; "+getId("midx")+
+      cb.append("for (int "+getId("midx")+" = 0; "+getId("midx")+
           " < "+getId("mi1")+" && "+getId("midx")+" < "+getId("mi2")+
           "; "+getId("midx")+"++) {");
-      sb.append(mKey.genJavaCompareBytes());
-      sb.append(mValue.genJavaSlurpBytes("b1", "s1", "l1"));
-      sb.append(mValue.genJavaSlurpBytes("b2", "s2", "l2"));
-      sb.append("           }\n");
-      sb.append("           if ("+getId("mi1")+" != "+getId("mi2")+
+      key.genCompareBytes(cb);
+      value.genSlurpBytes(cb, "b1", "s1", "l1");
+      value.genSlurpBytes(cb, "b2", "s2", "l2");
+      cb.append("}\n");
+      cb.append("if ("+getId("mi1")+" != "+getId("mi2")+
           ") { return ("+getId("mi1")+"<"+getId("mi2")+")?-1:0; }\n");
       decrLevel();
-      sb.append("        }\n");
-      return sb.toString();
+      cb.append("}\n");
     }
+  }
+  
+  /** Creates a new instance of JMap */
+  public JMap(JType t1, JType t2) {
+    setJavaType(new JavaMap(t1.getJavaType(), t2.getJavaType()));
+    setCppType(new CppType(" ::std::map<"+t1.getCppType().getType()+","+
+        t2.getCppType().getType()+">"));
+    setCType(new CType());
+    keyType = t1;
+    valueType = t2;
+  }
+  
+  String getSignature() {
+    return "{" + keyType.getSignature() + valueType.getSignature() +"}";
+  }
 }

src/java/org/apache/hadoop/record/compiler/JRecord.java

@@ -29,391 +29,451 @@ import java.util.Iterator;
  * @author Milind Bhandarkar
  */
 public class JRecord extends JCompType {
-
-    private String mFQName;
-    private String mName;
-    private String mModule;
-    private ArrayList mFields;
-    
-    /**
-     * Creates a new instance of JRecord
-     */
-    public JRecord(String name, ArrayList flist) {
-        super(name.replaceAll("\\.","::"), name, "Record", name);
-        mFQName = name;
-        int idx = name.lastIndexOf('.');
-        mName = name.substring(idx+1);
-        mModule = name.substring(0, idx);
-        mFields = flist;
-    }
-    
-    public String getName() {
-        return mName;
-    }
+  
+  class JavaRecord extends JavaCompType {
     
-    public String getJavaFQName() {
-        return mFQName;
-    }
-    
-    public String getCppFQName() {
-        return mFQName.replaceAll("\\.", "::");
-    }
+    private String fullName;
+    private String name;
+    private String module;
+    private ArrayList<JField<JavaType>> fields =
+        new ArrayList<JField<JavaType>>();
     
-    public String getJavaPackage() {
-        return mModule;
+    JavaRecord(String name, ArrayList<JField<JType>> flist) {
+      super(name, "Record", name);
+      this.fullName = name;
+      int idx = name.lastIndexOf('.');
+      this.name = name.substring(idx+1);
+      this.module = name.substring(0, idx);
+      for (Iterator<JField<JType>> iter = flist.iterator(); iter.hasNext();) {
+        JField<JType> f = iter.next();
+        fields.add(new JField<JavaType>(f.getName(), f.getType().getJavaType()));
+      }
     }
     
-    public String getCppNameSpace() {
-        return mModule.replaceAll("\\.", "::");
+    void genReadMethod(CodeBuffer cb, String fname, String tag, boolean decl) {
+      if (decl) {
+        cb.append(fullName+" "+fname+";\n");
+      }
+      cb.append(fname+"= new "+fullName+"();\n");
+      cb.append("a_.readRecord("+fname+",\""+tag+"\");\n");
     }
     
-    public ArrayList getFields() {
-        return mFields;
+    void genWriteMethod(CodeBuffer cb, String fname, String tag) {
+      cb.append("a_.writeRecord("+fname+",\""+tag+"\");\n");
     }
     
-    public String getSignature() {
-        StringBuffer sb = new StringBuffer();
-        sb.append("L").append(mName).append("(");
-        for (Iterator i = mFields.iterator(); i.hasNext();) {
-            String s = ((JField) i.next()).getSignature();
-            sb.append(s);
-        }
-        sb.append(")");
-        return sb.toString();
-    }
-    
-    public String genCppDecl(String fname) {
-        return "  "+mName+" "+fname+";\n";
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("int r = "+fullName+
+          ".Comparator.slurpRaw("+b+","+s+","+l+");\n");
+      cb.append(s+"+=r; "+l+"-=r;\n");
+      cb.append("}\n");
     }
     
-    public String genJavaReadMethod(String fname, String tag) {
-        return genJavaReadWrapper(fname, tag, false);
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("int r1 = "+fullName+
+          ".Comparator.compareRaw(b1,s1,l1,b2,s2,l2);\n");
+      cb.append("if (r1 <= 0) { return r1; }\n");
+      cb.append("s1+=r1; s2+=r1; l1-=r1; l2-=r1;\n");
+      cb.append("}\n");
     }
     
-    public String genJavaReadWrapper(String fname, String tag, boolean decl) {
-        StringBuffer ret = new StringBuffer("");
-        if (decl) {
-            ret.append("    "+getJavaFQName()+" "+fname+";\n");
+    void genCode(String destDir, ArrayList<String> options) throws IOException {
+      String pkg = module;
+      String pkgpath = pkg.replaceAll("\\.", "/");
+      File pkgdir = new File(destDir, pkgpath);
+      if (!pkgdir.exists()) {
+        // create the pkg directory
+        boolean ret = pkgdir.mkdirs();
+        if (!ret) {
+          throw new IOException("Cannnot create directory: "+pkgpath);
         }
-        ret.append("    "+fname+"= new "+getJavaFQName()+"();\n");
-        ret.append("    a_.readRecord("+fname+",\""+tag+"\");\n");
-        return ret.toString();
+      } else if (!pkgdir.isDirectory()) {
+        // not a directory
+        throw new IOException(pkgpath+" is not a directory.");
+      }
+      File jfile = new File(pkgdir, name+".java");
+      FileWriter jj = new FileWriter(jfile);
+      
+      CodeBuffer cb = new CodeBuffer();
+      cb.append("// File generated by hadoop record compiler. Do not edit.\n");
+      cb.append("package "+module+";\n\n");
+      cb.append("public class "+name+
+          " implements org.apache.hadoop.record.Record");
+      cb.append(", org.apache.hadoop.io.WritableComparable");
+      cb.append(" {\n");
+      
+      for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
+        JField<JavaType> jf = i.next();
+        String name = jf.getName();
+        JavaType type = jf.getType();
+        type.genDecl(cb, name);
+      }
+      cb.append("public "+name+"() { }\n");
+      
+      
+      cb.append("public "+name+"(\n");
+      int fIdx = 0;
+      for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext(); fIdx++) {
+        JField<JavaType> jf = i.next();
+        String name = jf.getName();
+        JavaType type = jf.getType();
+        type.genConstructorParam(cb, name);
+        cb.append((!i.hasNext())?"":",\n");
+      }
+      cb.append(") {\n");
+      fIdx = 0;
+      for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext(); fIdx++) {
+        JField<JavaType> jf = i.next();
+        String name = jf.getName();
+        JavaType type = jf.getType();
+        type.genConstructorSet(cb, name);
+      }
+      cb.append("}\n");
+      for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
+        JField<JavaType> jf = i.next();
+        String name = jf.getName();
+        JavaType type = jf.getType();
+        type.genGetSet(cb, name);
+      }
+      cb.append("public void serialize("+
+          "final org.apache.hadoop.record.OutputArchive a_, final String tag)\n"+
+          "throws java.io.IOException {\n");
+      cb.append("a_.startRecord(this,tag);\n");
+      for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
+        JField<JavaType> jf = i.next();
+        String name = jf.getName();
+        JavaType type = jf.getType();
+        type.genWriteMethod(cb, name, name);
+      }
+      cb.append("a_.endRecord(this,tag);\n");
+      cb.append("}\n");
+      
+      cb.append("public void deserialize("+
+          "final org.apache.hadoop.record.InputArchive a_, final String tag)\n"+
+          "throws java.io.IOException {\n");
+      cb.append("a_.startRecord(tag);\n");
+      for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
+        JField<JavaType> jf = i.next();
+        String name = jf.getName();
+        JavaType type = jf.getType();
+        type.genReadMethod(cb, name, name, false);
+      }
+      cb.append("a_.endRecord(tag);\n");
+      cb.append("}\n");
+      
+      cb.append("public String toString() {\n");
+      cb.append("try {\n");
+      cb.append("java.io.ByteArrayOutputStream s =\n");
+      cb.append("  new java.io.ByteArrayOutputStream();\n");
+      cb.append("org.apache.hadoop.record.CsvOutputArchive a_ = \n");
+      cb.append("  new org.apache.hadoop.record.CsvOutputArchive(s);\n");
+      cb.append("this.serialize(a_,\"\");\n");
+      cb.append("return new String(s.toByteArray(), \"UTF-8\");\n");
+      cb.append("} catch (Throwable ex) {\n");
+      cb.append("throw new RuntimeException(ex);\n");
+      cb.append("}\n");
+      cb.append("}\n");
+      
+      cb.append("public void write(final java.io.DataOutput out)\n"+
+          "throws java.io.IOException {\n");
+      cb.append("org.apache.hadoop.record.BinaryOutputArchive archive =\n"+
+          "new org.apache.hadoop.record.BinaryOutputArchive(out);\n");
+      cb.append("this.serialize(archive, \"\");\n");
+      cb.append("}\n");
+      
+      cb.append("public void readFields(final java.io.DataInput in)\n"+
+          "throws java.io.IOException {\n");
+      cb.append("org.apache.hadoop.record.BinaryInputArchive archive =\n"+
+          "new org.apache.hadoop.record.BinaryInputArchive(in);\n");
+      cb.append("this.deserialize(archive, \"\");\n");
+      cb.append("}\n");
+      cb.append("public int compareTo (final Object peer_) throws ClassCastException {\n");
+      cb.append("if (!(peer_ instanceof "+name+")) {\n");
+      cb.append("throw new ClassCastException(\"Comparing different types of records.\");\n");
+      cb.append("}\n");
+      cb.append(name+" peer = ("+name+") peer_;\n");
+      cb.append("int ret = 0;\n");
+      for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
+        JField<JavaType> jf = i.next();
+        String name = jf.getName();
+        JavaType type = jf.getType();
+        type.genCompareTo(cb, name, "peer."+name);
+        cb.append("if (ret != 0) return ret;\n");
+      }
+      cb.append("return ret;\n");
+      cb.append("}\n");
+      
+      cb.append("public boolean equals(final Object peer_) {\n");
+      cb.append("if (!(peer_ instanceof "+name+")) {\n");
+      cb.append("return false;\n");
+      cb.append("}\n");
+      cb.append("if (peer_ == this) {\n");
+      cb.append("return true;\n");
+      cb.append("}\n");
+      cb.append(name+" peer = ("+name+") peer_;\n");
+      cb.append("boolean ret = false;\n");
+      for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
+        JField<JavaType> jf = i.next();
+        String name = jf.getName();
+        JavaType type = jf.getType();
+        type.genEquals(cb, name, "peer."+name);
+        cb.append("if (!ret) return ret;\n");
+      }
+      cb.append("return ret;\n");
+      cb.append("}\n");
+      
+      cb.append("public Object clone() throws CloneNotSupportedException {\n");
+      cb.append(name+" other = new "+name+"();\n");
+      for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
+        JField<JavaType> jf = i.next();
+        String name = jf.getName();
+        JavaType type = jf.getType();
+        type.genClone(cb, name);
+      }
+      cb.append("return other;\n");
+      cb.append("}\n");
+      
+      cb.append("public int hashCode() {\n");
+      cb.append("int result = 17;\n");
+      cb.append("int ret;\n");
+      for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
+        JField<JavaType> jf = i.next();
+        String name = jf.getName();
+        JavaType type = jf.getType();
+        type.genHashCode(cb, name);
+        cb.append("result = 37*result + ret;\n");
+      }
+      cb.append("return result;\n");
+      cb.append("}\n");
+      
+      cb.append("public static String signature() {\n");
+      cb.append("return \""+getSignature()+"\";\n");
+      cb.append("}\n");
+      
+      cb.append("public static class Comparator extends"+
+          " org.apache.hadoop.io.WritableComparator {\n");
+      cb.append("public Comparator() {\n");
+      cb.append("super("+name+".class);\n");
+      cb.append("}\n");
+      
+      cb.append("static public int slurpRaw(byte[] b, int s, int l) {\n");
+      cb.append("try {\n");
+      cb.append("int os = s;\n");
+      for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
+        JField<JavaType> jf = i.next();
+        String name = jf.getName();
+        JavaType type = jf.getType();
+        type.genSlurpBytes(cb, "b","s","l");
+      }
+      cb.append("return (os - s);\n");
+      cb.append("} catch(java.io.IOException e) {\n");
+      cb.append("throw new RuntimeException(e);\n");
+      cb.append("}\n");
+      cb.append("}\n");
+      
+      cb.append("static public int compareRaw(byte[] b1, int s1, int l1,\n");
+      cb.append("                             byte[] b2, int s2, int l2) {\n");
+      cb.append("try {\n");
+      cb.append("int os1 = s1;\n");
+      for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
+        JField<JavaType> jf = i.next();
+        String name = jf.getName();
+        JavaType type = jf.getType();
+        type.genCompareBytes(cb);
+      }
+      cb.append("return (os1 - s1);\n");
+      cb.append("} catch(java.io.IOException e) {\n");
+      cb.append("throw new RuntimeException(e);\n");
+      cb.append("}\n");
+      cb.append("}\n");
+      cb.append("public int compare(byte[] b1, int s1, int l1,\n");
+      cb.append("                   byte[] b2, int s2, int l2) {\n");
+      cb.append("int ret = compareRaw(b1,s1,l1,b2,s2,l2);\n");
+      cb.append("return (ret == -1)? -1 : ((ret==0)? 1 : 0);");
+      cb.append("}\n");
+      cb.append("}\n\n");
+      cb.append("static {\n");
+      cb.append("org.apache.hadoop.io.WritableComparator.define("
+          +name+".class, new Comparator());\n");
+      cb.append("}\n");
+      cb.append("}\n");
+
+      jj.write(cb.toString());
+      jj.close();
     }
+  }
+  
+  class CppRecord extends CppCompType {
     
-    public String genJavaWriteWrapper(String fname, String tag) {
-        return "    a_.writeRecord("+fname+",\""+tag+"\");\n";
-    }
+    private String fullName;
+    private String name;
+    private String module;
+    private ArrayList<JField<CppType>> fields = 
+        new ArrayList<JField<CppType>>();
     
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           int r = "+getJavaFQName()+
-          ".Comparator.slurpRaw("+b+","+s+","+l+");\n");
-      sb.append("           "+s+"+=r; "+l+"-=r;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    CppRecord(String name, ArrayList<JField<JType>> flist) {
+      super(name.replaceAll("\\.","::"));
+      this.fullName = name.replaceAll("\\.", "::");
+      int idx = name.lastIndexOf('.');
+      this.name = name.substring(idx+1);
+      this.module = name.substring(0, idx).replaceAll("\\.", "::");
+      for (Iterator<JField<JType>> iter = flist.iterator(); iter.hasNext();) {
+        JField<JType> f = iter.next();
+        fields.add(new JField<CppType>(f.getName(), f.getType().getCppType()));
+      }
     }
     
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           int r1 = "+getJavaFQName()+
-          ".Comparator.compareRaw(b1,s1,l1,b2,s2,l2);\n");
-      sb.append("           if (r1 <= 0) { return r1; }\n");
-      sb.append("           s1+=r1; s2+=r1; l1-=r1; l2-=r1;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    String genDecl(String fname) {
+      return "  "+name+" "+fname+";\n";
     }
     
-    public void genCppCode(FileWriter hh, FileWriter cc)
-        throws IOException {
-        String[] ns = getCppNameSpace().split("::");
-        for (int i = 0; i < ns.length; i++) {
-            hh.write("namespace "+ns[i]+" {\n");
-        }
-        
-        hh.write("class "+getName()+" : public ::hadoop::Record {\n");
-        hh.write("private:\n");
-        
-        for (Iterator i = mFields.iterator(); i.hasNext();) {
-            JField jf = (JField) i.next();
-            hh.write(jf.genCppDecl());
-        }
-        hh.write("public:\n");
-        hh.write("  virtual void serialize(::hadoop::OArchive& a_, const char* tag) const;\n");
-        hh.write("  virtual void deserialize(::hadoop::IArchive& a_, const char* tag);\n");
-        hh.write("  virtual const ::std::string& type() const;\n");
-        hh.write("  virtual const ::std::string& signature() const;\n");
-        hh.write("  virtual bool operator<(const "+getName()+"& peer_) const;\n");
-        hh.write("  virtual bool operator==(const "+getName()+"& peer_) const;\n");
-        hh.write("  virtual ~"+getName()+"() {};\n");
-        int fIdx = 0;
-        for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
-            JField jf = (JField) i.next();
-            hh.write(jf.genCppGetSet(fIdx));
+    void genCode(FileWriter hh, FileWriter cc, ArrayList<String> options)
+    throws IOException {
+      CodeBuffer hb = new CodeBuffer();
+      
+      String[] ns = module.split("::");
+      for (int i = 0; i < ns.length; i++) {
+        hb.append("namespace "+ns[i]+" {\n");
+      }
+      
+      hb.append("class "+name+" : public ::hadoop::Record {\n");
+      hb.append("private:\n");
+      
+      for (Iterator<JField<CppType>> i = fields.iterator(); i.hasNext();) {
+        JField<CppType> jf = i.next();
+        String name = jf.getName();
+        CppType type = jf.getType();
+        type.genDecl(hb, name);
+      }
+      hb.append("public:\n");
+      hb.append("virtual void serialize(::hadoop::OArchive& a_, const char* tag) const;\n");
+      hb.append("virtual void deserialize(::hadoop::IArchive& a_, const char* tag);\n");
+      hb.append("virtual const ::std::string& type() const;\n");
+      hb.append("virtual const ::std::string& signature() const;\n");
+      hb.append("virtual bool operator<(const "+name+"& peer_) const;\n");
+      hb.append("virtual bool operator==(const "+name+"& peer_) const;\n");
+      hb.append("virtual ~"+name+"() {};\n");
+      for (Iterator<JField<CppType>> i = fields.iterator(); i.hasNext();) {
+        JField<CppType> jf = i.next();
+        String name = jf.getName();
+        CppType type = jf.getType();
+        type.genGetSet(hb, name);
+      }
+      hb.append("}; // end record "+name+"\n");
+      for (int i=ns.length-1; i>=0; i--) {
+        hb.append("} // end namespace "+ns[i]+"\n");
+      }
+      
+      hh.write(hb.toString());
+      
+      CodeBuffer cb = new CodeBuffer();
+      
+      cb.append("void "+fullName+"::serialize(::hadoop::OArchive& a_, const char* tag) const {\n");
+      cb.append("a_.startRecord(*this,tag);\n");
+      for (Iterator<JField<CppType>> i = fields.iterator(); i.hasNext();) {
+        JField<CppType> jf = i.next();
+        String name = jf.getName();
+        CppType type = jf.getType();
+        if (type instanceof JBuffer.CppBuffer) {
+          cb.append("a_.serialize("+name+","+name+".length(),\""+name+"\");\n");
+        } else {
+          cb.append("a_.serialize("+name+",\""+name+"\");\n");
         }
-        hh.write("}; // end record "+getName()+"\n");
-        for (int i=ns.length-1; i>=0; i--) {
-            hh.write("} // end namespace "+ns[i]+"\n");
+      }
+      cb.append("a_.endRecord(*this,tag);\n");
+      cb.append("return;\n");
+      cb.append("}\n");
+      
+      cb.append("void "+fullName+"::deserialize(::hadoop::IArchive& a_, const char* tag) {\n");
+      cb.append("a_.startRecord(*this,tag);\n");
+      for (Iterator<JField<CppType>> i = fields.iterator(); i.hasNext();) {
+        JField<CppType> jf = i.next();
+        String name = jf.getName();
+        CppType type = jf.getType();
+        if (type instanceof JBuffer.CppBuffer) {
+          cb.append("{\nsize_t len=0; a_.deserialize("+name+",len,\""+name+"\");\n}\n");
+        } else {
+          cb.append("a_.deserialize("+name+",\""+name+"\");\n");
         }
-        cc.write("void "+getCppFQName()+"::serialize(::hadoop::OArchive& a_, const char* tag) const {\n");
-        cc.write("  a_.startRecord(*this,tag);\n");
-        fIdx = 0;
-        for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
-            JField jf = (JField) i.next();
-            String name = jf.getName();
-            if (jf.getType() instanceof JBuffer) {
-                cc.write("  a_.serialize("+name+","+name+".length(),\""+jf.getTag()+"\");\n");
-            } else {
-                cc.write("  a_.serialize("+name+",\""+jf.getTag()+"\");\n");
-            }
-        }
-        cc.write("  a_.endRecord(*this,tag);\n");
-        cc.write("  return;\n");
-        cc.write("}\n");
-        
-        cc.write("void "+getCppFQName()+"::deserialize(::hadoop::IArchive& a_, const char* tag) {\n");
-        cc.write("  a_.startRecord(*this,tag);\n");
-        fIdx = 0;
-        for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
-            JField jf = (JField) i.next();
-            String name = jf.getName();
-            if (jf.getType() instanceof JBuffer) {
-                cc.write("  { size_t len=0; a_.deserialize("+name+",len,\""+jf.getTag()+"\");}\n");
-            } else {
-                cc.write("  a_.deserialize("+name+",\""+jf.getTag()+"\");\n");
-            }
-        }
-        cc.write("  a_.endRecord(*this,tag);\n");
-        cc.write("  return;\n");
-        cc.write("}\n");
-        
-        
-        cc.write("bool "+getCppFQName()+"::operator< (const "+getCppFQName()+"& peer_) const {\n");
-        cc.write("  return (1\n");
-        for (Iterator i = mFields.iterator(); i.hasNext();) {
-            JField jf = (JField) i.next();
-            String name = jf.getName();
-            cc.write("    && ("+name+" < peer_."+name+")\n");
-        }
-        cc.write("  );\n");
-        cc.write("}\n");
-        
-        cc.write("bool "+getCppFQName()+"::operator== (const "+getCppFQName()+"& peer_) const {\n");
-        cc.write("  return (1\n");
-        for (Iterator i = mFields.iterator(); i.hasNext();) {
-            JField jf = (JField) i.next();
-            String name = jf.getName();
-            cc.write("    && ("+name+" == peer_."+name+")\n");
-        }
-        cc.write("  );\n");
-        cc.write("}\n");
-        
-        cc.write("const ::std::string&"+getCppFQName()+"::type() const {\n");
-        cc.write("  static const ::std::string type_(\""+mName+"\");\n");
-        cc.write("  return type_;\n");
-        cc.write("}\n");
-        
-        cc.write("const ::std::string&"+getCppFQName()+"::signature() const {\n");
-        cc.write("  static const ::std::string sig_(\""+getSignature()+"\");\n");
-        cc.write("  return sig_;\n");
-        cc.write("}\n");
-        
+      }
+      cb.append("a_.endRecord(*this,tag);\n");
+      cb.append("return;\n");
+      cb.append("}\n");
+      
+      
+      cb.append("bool "+fullName+"::operator< (const "+fullName+"& peer_) const {\n");
+      cb.append("return (1\n");
+      for (Iterator<JField<CppType>> i = fields.iterator(); i.hasNext();) {
+        JField<CppType> jf = i.next();
+        String name = jf.getName();
+        cb.append("&& ("+name+" < peer_."+name+")\n");
+      }
+      cb.append(");\n");
+      cb.append("}\n");
+      
+      cb.append("bool "+fullName+"::operator== (const "+fullName+"& peer_) const {\n");
+      cb.append("return (1\n");
+      for (Iterator<JField<CppType>> i = fields.iterator(); i.hasNext();) {
+        JField<CppType> jf = i.next();
+        String name = jf.getName();
+        cb.append("&& ("+name+" == peer_."+name+")\n");
+      }
+      cb.append(");\n");
+      cb.append("}\n");
+      
+      cb.append("const ::std::string&"+fullName+"::type() const {\n");
+      cb.append("static const ::std::string type_(\""+name+"\");\n");
+      cb.append("return type_;\n");
+      cb.append("}\n");
+      
+      cb.append("const ::std::string&"+fullName+"::signature() const {\n");
+      cb.append("static const ::std::string sig_(\""+getSignature()+"\");\n");
+      cb.append("return sig_;\n");
+      cb.append("}\n");
+      
+      cc.write(cb.toString());
     }
+  }
+  
+  class CRecord extends CCompType {
     
-    public void genJavaCode(String destDir) throws IOException {
-        String pkg = getJavaPackage();
-        String pkgpath = pkg.replaceAll("\\.", "/");
-        File pkgdir = new File(destDir, pkgpath);
-        if (!pkgdir.exists()) {
-            // create the pkg directory
-            boolean ret = pkgdir.mkdirs();
-            if (!ret) {
-                throw new IOException("Cannnot create directory: "+pkgpath);
-            }
-        } else if (!pkgdir.isDirectory()) {
-            // not a directory
-            throw new IOException(pkgpath+" is not a directory.");
-        }
-        File jfile = new File(pkgdir, getName()+".java");
-        FileWriter jj = new FileWriter(jfile);
-        jj.write("// File generated by hadoop record compiler. Do not edit.\n");
-        jj.write("package "+getJavaPackage()+";\n\n");
-        jj.write("import java.io.IOException;\n");
-        jj.write("import org.apache.commons.logging.Log;\n");
-        jj.write("import org.apache.commons.logging.LogFactory;\n");
-        jj.write("import org.apache.hadoop.io.WritableComparator;\n");
-        jj.write("import org.apache.hadoop.io.WritableComparable;\n");
-        jj.write("import org.apache.hadoop.io.WritableUtils;\n");
-        jj.write("import org.apache.hadoop.io.BytesWritable;\n");
-        jj.write("import org.apache.hadoop.io.Text;\n\n");
-        jj.write("public class "+getName()+" implements org.apache.hadoop.record.Record, WritableComparable {\n");
-        jj.write("  private static final Log LOG= LogFactory.getLog(\""+
-            this.getJavaFQName()+"\");\n");
-        for (Iterator i = mFields.iterator(); i.hasNext();) {
-            JField jf = (JField) i.next();
-            jj.write(jf.genJavaDecl());
-        }
-        jj.write("  public "+getName()+"() { }\n");
-        
-        
-        jj.write("  public "+getName()+"(\n");
-        int fIdx = 0;
-        for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
-            JField jf = (JField) i.next();
-            jj.write(jf.genJavaConstructorParam(fIdx));
-            jj.write((!i.hasNext())?"":",\n");
-        }
-        jj.write(") {\n");
-        fIdx = 0;
-        for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
-            JField jf = (JField) i.next();
-            jj.write(jf.genJavaConstructorSet(fIdx));
-        }
-        jj.write("  }\n");
-        fIdx = 0;
-        for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
-            JField jf = (JField) i.next();
-            jj.write(jf.genJavaGetSet(fIdx));
-        }
-        jj.write("  public void serialize(org.apache.hadoop.record.OutputArchive a_, String tag) throws java.io.IOException {\n");
-        jj.write("    a_.startRecord(this,tag);\n");
-        for (Iterator i = mFields.iterator(); i.hasNext();) {
-            JField jf = (JField) i.next();
-            jj.write(jf.genJavaWriteMethodName());
-        }
-        jj.write("    a_.endRecord(this,tag);\n");
-        jj.write("  }\n");
-        
-        jj.write("  public void deserialize(org.apache.hadoop.record.InputArchive a_, String tag) throws java.io.IOException {\n");
-        jj.write("    a_.startRecord(tag);\n");
-        for (Iterator i = mFields.iterator(); i.hasNext();) {
-            JField jf = (JField) i.next();
-            jj.write(jf.genJavaReadMethodName());
-        }
-        jj.write("    a_.endRecord(tag);\n");
-        jj.write("}\n");
-        
-        jj.write("  public String toString() {\n");
-        jj.write("    try {\n");
-        jj.write("      java.io.ByteArrayOutputStream s =\n");
-        jj.write("        new java.io.ByteArrayOutputStream();\n");
-        jj.write("      org.apache.hadoop.record.CsvOutputArchive a_ = \n");
-        jj.write("        new org.apache.hadoop.record.CsvOutputArchive(s);\n");
-        jj.write("      a_.startRecord(this,\"\");\n");
-        fIdx = 0;
-        for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
-            JField jf = (JField) i.next();
-            jj.write(jf.genJavaWriteMethodName());
-        }
-        jj.write("      a_.endRecord(this,\"\");\n");
-        jj.write("      return new String(s.toByteArray(), \"UTF-8\");\n");
-        jj.write("    } catch (Throwable ex) {\n");
-        jj.write("      throw new RuntimeException(ex);\n");
-        jj.write("    }\n");
-        jj.write("  }\n");
-        
-        jj.write("  public void write(java.io.DataOutput out) throws java.io.IOException {\n");
-        jj.write("    org.apache.hadoop.record.BinaryOutputArchive archive = new org.apache.hadoop.record.BinaryOutputArchive(out);\n");
-        jj.write("    serialize(archive, \"\");\n");
-        jj.write("  }\n");
-        
-        jj.write("  public void readFields(java.io.DataInput in) throws java.io.IOException {\n");
-        jj.write("    org.apache.hadoop.record.BinaryInputArchive archive = new org.apache.hadoop.record.BinaryInputArchive(in);\n");
-        jj.write("    deserialize(archive, \"\");\n");
-        jj.write("  }\n");
-        
-        jj.write("  public int compareTo (Object peer_) throws ClassCastException {\n");
-        jj.write("    if (!(peer_ instanceof "+getName()+")) {\n");
-        jj.write("      throw new ClassCastException(\"Comparing different types of records.\");\n");
-        jj.write("    }\n");
-        jj.write("    "+getName()+" peer = ("+getName()+") peer_;\n");
-        jj.write("    int ret = 0;\n");
-        for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
-            JField jf = (JField) i.next();
-            jj.write(jf.genJavaCompareTo());
-            jj.write("    if (ret != 0) return ret;\n");
-        }
-        jj.write("     return ret;\n");
-        jj.write("  }\n");
-        
-        jj.write("  public boolean equals(Object peer_) {\n");
-        jj.write("    if (!(peer_ instanceof "+getName()+")) {\n");
-        jj.write("      return false;\n");
-        jj.write("    }\n");
-        jj.write("    if (peer_ == this) {\n");
-        jj.write("      return true;\n");
-        jj.write("    }\n");
-        jj.write("    "+getName()+" peer = ("+getName()+") peer_;\n");
-        jj.write("    boolean ret = false;\n");
-        for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
-            JField jf = (JField) i.next();
-            jj.write(jf.genJavaEquals());
-            jj.write("    if (!ret) return ret;\n");
-        }
-        jj.write("     return ret;\n");
-        jj.write("  }\n");
-        
-        jj.write("  public int hashCode() {\n");
-        jj.write("    int result = 17;\n");
-        jj.write("    int ret;\n");
-        for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
-            JField jf = (JField) i.next();
-            jj.write(jf.genJavaHashCode());
-            jj.write("    result = 37*result + ret;\n");
-        }
-        jj.write("    return result;\n");
-        jj.write("  }\n");
-        jj.write("  public static String signature() {\n");
-        jj.write("    return \""+getSignature()+"\";\n");
-        jj.write("  }\n");
-        
-        jj.write("  public static class Comparator extends WritableComparator {\n");
-        jj.write("    public Comparator() {\n");
-        jj.write("      super("+getName()+".class);\n");
-        jj.write("    }\n");
-
-        jj.write("    static public int slurpRaw(byte[] b, int s, int l) {\n");
-        jj.write("      try {\n");
-        jj.write("        int os = s;\n");
-        for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
-            JField jf = (JField) i.next();
-            jj.write(jf.genJavaSlurpBytes("b","s","l"));
-        }
-        jj.write("        return (os - s);\n");
-        jj.write("      } catch(IOException e) {\n");
-        jj.write("        LOG.warn(e);\n");
-        jj.write("        throw new RuntimeException(e);\n");
-        jj.write("      }\n");
-        jj.write("    }\n");
-        
-        jj.write("    static public int compareRaw(byte[] b1, int s1, int l1,\n");
-        jj.write("                       byte[] b2, int s2, int l2) {\n");
-        jj.write("      try {\n");
-        jj.write("        int os1 = s1;\n");
-        for (Iterator i = mFields.iterator(); i.hasNext(); fIdx++) {
-            JField jf = (JField) i.next();
-            jj.write(jf.genJavaCompareBytes());
-        }
-        jj.write("        return (os1 - s1);\n");
-        jj.write("      } catch(IOException e) {\n");
-        jj.write("        LOG.warn(e);\n");
-        jj.write("        throw new RuntimeException(e);\n");
-        jj.write("      }\n");
-        jj.write("    }\n");
-        jj.write("    public int compare(byte[] b1, int s1, int l1,\n");
-        jj.write("                       byte[] b2, int s2, int l2) {\n");
-        jj.write("      int ret = compareRaw(b1,s1,l1,b2,s2,l2);\n");
-        jj.write("      return (ret == -1)? -1 : ((ret==0)? 1 : 0);");
-        jj.write("    }\n");
-        jj.write("  }\n\n");
-        jj.write("  static {\n");
-        jj.write("    WritableComparator.define("+getName()+".class, new Comparator());\n");
-        jj.write("  }\n");
-
-        
-        jj.write("}\n");
-        
-        jj.close();
+  }
+  
+  private String signature;
+  
+  /**
+   * Creates a new instance of JRecord
+   */
+  public JRecord(String name, ArrayList<JField<JType>> flist) {
+    setJavaType(new JavaRecord(name, flist));
+    setCppType(new CppRecord(name, flist));
+    setCType(new CRecord());
+    // precompute signature
+    int idx = name.lastIndexOf('.');
+    String recName = name.substring(idx+1);
+    StringBuffer sb = new StringBuffer();
+    sb.append("L").append(recName).append("(");
+    for (Iterator<JField<JType>> i = flist.iterator(); i.hasNext();) {
+      String s = i.next().getType().getSignature();
+      sb.append(s);
     }
+    sb.append(")");
+    signature = sb.toString();
+  }
+  
+  String getSignature() {
+    return signature;
+  }
+  
+  void genCppCode(FileWriter hh, FileWriter cc, ArrayList<String> options)
+    throws IOException {
+    ((CppRecord)getCppType()).genCode(hh,cc, options);
+  }
+  
+  void genJavaCode(String destDir, ArrayList<String> options)
+  throws IOException {
+    ((JavaRecord)getJavaType()).genCode(destDir, options);
+  }
 }

src/java/org/apache/hadoop/record/compiler/JString.java

@@ -18,55 +18,54 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JCompType.CCompType;
+import org.apache.hadoop.record.compiler.JCompType.CppCompType;
+
 /**
  *
  * @author Milind Bhandarkar
  */
 public class JString extends JCompType {
     
-    /** Creates a new instance of JString */
-    public JString() {
-        super(" ::std::string", "Text", "String", "Text");
-    }
+  class JavaString extends JavaCompType {
     
-    public String getSignature() {
-        return "s";
+    JavaString() {
+      super("String", "String", "String");
     }
     
-    public String genJavaReadWrapper(String fname, String tag, boolean decl) {
-        String ret = "";
-        if (decl) {
-            ret = "    Text "+fname+";\n";
-        }
-        return ret + "        "+fname+"=a_.readString(\""+tag+"\");\n";
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("int i = org.apache.hadoop.record.Utils.readVInt("+b+", "+s+");\n");
+      cb.append("int z = org.apache.hadoop.record.Utils.getVIntSize(i);\n");
+      cb.append(s+"+=(z+i); "+l+"-= (z+i);\n");
+      cb.append("}\n");
     }
     
-    public String genJavaWriteWrapper(String fname, String tag) {
-        return "        a_.writeString("+fname+",\""+tag+"\");\n";
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("int i1 = org.apache.hadoop.record.Utils.readVInt(b1, s1);\n");
+      cb.append("int i2 = org.apache.hadoop.record.Utils.readVInt(b2, s2);\n");
+      cb.append("int z1 = org.apache.hadoop.record.Utils.getVIntSize(i1);\n");
+      cb.append("int z2 = org.apache.hadoop.record.Utils.getVIntSize(i2);\n");
+      cb.append("s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
+      cb.append("int r1 = org.apache.hadoop.record.Utils.compareBytes(b1,s1,l1,b2,s2,l2);\n");
+      cb.append("if (r1 != 0) { return (r1<0)?-1:0; }\n");
+      cb.append("s1+=i1; s2+=i2; l1-=i1; l1-=i2;\n");
+      cb.append("}\n");
     }
     
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           int i = WritableComparator.readVInt("+b+", "+s+");\n");
-      sb.append("           int z = WritableUtils.getVIntSize(i);\n");
-      sb.append("           "+s+"+=(z+i); "+l+"-= (z+i);\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genClone(CodeBuffer cb, String fname) {
+      cb.append("other."+fname+" = this."+fname+";\n");
+    }
+  }
+    /** Creates a new instance of JString */
+    public JString() {
+      setJavaType(new JavaString());
+      setCppType(new CppCompType(" ::std::string"));
+      setCType(new CCompType());
     }
     
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           int i1 = WritableComparator.readVInt(b1, s1);\n");
-      sb.append("           int i2 = WritableComparator.readVInt(b2, s2);\n");
-      sb.append("           int z1 = WritableUtils.getVIntSize(i1);\n");
-      sb.append("           int z2 = WritableUtils.getVIntSize(i2);\n");
-      sb.append("           s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
-      sb.append("           int r1 = WritableComparator.compareBytes(b1,s1,l1,b2,s2,l2);\n");
-      sb.append("           if (r1 != 0) { return (r1<0)?-1:0; }\n");
-      sb.append("           s1+=i1; s2+=i2; l1-=i1; l1-=i2;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    String getSignature() {
+        return "s";
     }
 }

src/java/org/apache/hadoop/record/compiler/JType.java

@@ -20,129 +20,154 @@ package org.apache.hadoop.record.compiler;
 
 /**
  * Abstract Base class for all types supported by Hadoop Record I/O.
- * 
+ *
  * @author Milind Bhandarkar
  */
 abstract public class JType {
-    
-    private String mCppName;
-    private String mJavaName;
-    private String mMethodSuffix;
-    private String mWrapper;
-    private String mUnwrapMethod;
-    
-    /**
-     * Creates a new instance of JType
-     */
-    JType(String cppname, String javaname, String suffix, String wrapper, String unwrap) {
-        mCppName = cppname;
-        mJavaName = javaname;
-        mMethodSuffix = suffix;
-        mWrapper = wrapper;
-        mUnwrapMethod = unwrap;
-    }
-    
-    abstract String getSignature();
-    
-    String genCppDecl(String fname) {
-        return "  "+mCppName+" m"+fname+";\n"; 
+  
+  static String toCamelCase(String name) {
+    char firstChar = name.charAt(0);
+    if (Character.isLowerCase(firstChar)) {
+      return ""+Character.toUpperCase(firstChar) + name.substring(1);
     }
+    return name;
+  }
+  
+  JavaType javaType;
+  CppType cppType;
+  CType cType;
+  
+  abstract class JavaType {
+    private String name;
+    private String methodSuffix;
+    private String wrapper;
     
-    String genJavaDecl (String fname) {
-        return "  private "+mJavaName+" m"+fname+";\n";
+    JavaType(String javaname,
+        String suffix,
+        String wrapper) {
+      this.name = javaname;
+      this.methodSuffix = suffix;
+      this.wrapper = wrapper;
     }
     
-    String genJavaConstructorParam (int fIdx) {
-        return "        "+mJavaName+" m"+fIdx;
+    void genDecl(CodeBuffer cb, String fname) {
+      cb.append("private "+name+" "+fname+";\n");
     }
     
-    String genCppGetSet(String fname, int fIdx) {
-        String getFunc = "  virtual "+mCppName+" get"+fname+"() const {\n";
-        getFunc += "    return m"+fname+";\n";
-        getFunc += "  }\n";
-        String setFunc = "  virtual void set"+fname+"("+mCppName+" m_) {\n";
-        setFunc += "    m"+fname+"=m_;\n";
-        setFunc += "  }\n";
-        return getFunc+setFunc;
+    void genConstructorParam(CodeBuffer cb, String fname) {
+      cb.append("final "+name+" "+fname);
     }
     
-    String genJavaGetSet(String fname, int fIdx) {
-        String getFunc = "  public "+mJavaName+" get"+fname+"() {\n";
-        getFunc += "    return m"+fname+";\n";
-        getFunc += "  }\n";
-        String setFunc = "  public void set"+fname+"("+mJavaName+" m_) {\n";
-        setFunc += "    m"+fname+"=m_;\n";
-        setFunc += "  }\n";
-        return getFunc+setFunc;
+    void genGetSet(CodeBuffer cb, String fname) {
+      cb.append("public "+name+" get"+toCamelCase(fname)+"() {\n");
+      cb.append("return "+fname+";\n");
+      cb.append("}\n");
+      cb.append("public void set"+toCamelCase(fname)+"(final "+name+" "+fname+") {\n");
+      cb.append("this."+fname+"="+fname+";\n");
+      cb.append("}\n");
     }
     
-    String getCppType() {
-        return mCppName;
+    String getType() {
+      return name;
     }
     
-    String getJavaType() {
-        return mJavaName;
-    }
-   
-    String getJavaWrapperType() {
-        return mWrapper;
+    String getWrapperType() {
+      return wrapper;
     }
     
     String getMethodSuffix() {
-        return mMethodSuffix;
+      return methodSuffix;
     }
     
-    String genJavaWriteMethod(String fname, String tag) {
-        return "    a_.write"+mMethodSuffix+"("+fname+",\""+tag+"\");\n";
+    void genWriteMethod(CodeBuffer cb, String fname, String tag) {
+      cb.append("a_.write"+methodSuffix+"("+fname+",\""+tag+"\");\n");
     }
     
-    String genJavaReadMethod(String fname, String tag) {
-        return "    "+fname+"=a_.read"+mMethodSuffix+"(\""+tag+"\");\n";
+    void genReadMethod(CodeBuffer cb, String fname, String tag, boolean decl) {
+      if (decl) {
+        cb.append(name+" "+fname+";\n");
+      }
+      cb.append(fname+"=a_.read"+methodSuffix+"(\""+tag+"\");\n");
     }
     
-    String genJavaReadWrapper(String fname, String tag, boolean decl) {
-        String ret = "";
-        if (decl) {
-            ret = "    "+mWrapper+" "+fname+";\n";
-        }
-        return ret + "    "+fname+"=new "+mWrapper+"(a_.read"+mMethodSuffix+"(\""+tag+"\"));\n";
+    void genCompareTo(CodeBuffer cb, String fname, String other) {
+      cb.append("ret = ("+fname+" == "+other+")? 0 :(("+fname+"<"+other+
+          ")?-1:1);\n");
     }
     
-    String genJavaWriteWrapper(String fname, String tag) {
-        return "        a_.write"+mMethodSuffix+"("+fname+"."+mUnwrapMethod+"(),\""+tag+"\");\n";
+    abstract void genCompareBytes(CodeBuffer cb);
+    
+    abstract void genSlurpBytes(CodeBuffer cb, String b, String s, String l);
+    
+    void genEquals(CodeBuffer cb, String fname, String peer) {
+      cb.append("ret = ("+fname+"=="+peer+");\n");
     }
     
-    String genJavaCompareToWrapper(String fname, String other) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("          "+mJavaName+" ee1 = ("+fname+"."+mUnwrapMethod+"();\n");
-      sb.append("          "+mJavaName+" ee2 = ("+other+"."+mUnwrapMethod+"();\n");
-      sb.append("          ret = (ee1 == ee2)? 0 :((ee1<ee2)?-1:1);\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genHashCode(CodeBuffer cb, String fname) {
+      cb.append("ret = (int)"+fname+";\n");
     }
     
-    String genJavaCompareTo(String fname, String other) {
-        return "    ret = ("+fname+" == "+other+")? 0 :(("+fname+"<"+other+")?-1:1);\n";
+    void genConstructorSet(CodeBuffer cb, String fname) {
+      cb.append("this."+fname+" = "+fname+";\n");
     }
     
-    String genJavaCompareBytes() {
-      return "        // throw new IOException(\"Not Implemented yet!\");\n";
+    void genClone(CodeBuffer cb, String fname) {
+      cb.append("other."+fname+" = this."+fname+";\n");
     }
+  }
+  
+  class CppType {
+    private String name;
     
-    String genJavaSlurpBytes(String b, String s, String l) {
-      return "        // throw new IOException(\"Not Implemented yet!\");\n";
+    CppType(String cppname) {
+      name = cppname;
     }
     
-    String genJavaEquals(String fname, String peer) {
-        return "    ret = ("+fname+"=="+peer+");\n";
+    void genDecl(CodeBuffer cb, String fname) {
+      cb.append(name+" "+fname+";\n");
     }
     
-    String genJavaHashCode(String fname) {
-        return "    ret = (int)"+fname+";\n";
+    void genGetSet(CodeBuffer cb, String fname) {
+      cb.append("virtual "+name+" get"+toCamelCase(fname)+"() const {\n");
+      cb.append("return "+fname+";\n");
+      cb.append("}\n");
+      cb.append("virtual void set"+toCamelCase(fname)+"("+name+" m_) {\n");
+      cb.append(fname+"=m_;\n");
+      cb.append("}\n");
     }
-
-    String genJavaConstructorSet(String fname, int fIdx) {
-        return "    m"+fname+"=m"+fIdx+";\n";
+    
+    String getType() {
+      return name;
     }
+  }
+  
+  class CType {
+    
+  }
+  
+  abstract String getSignature();
+  
+  void setJavaType(JavaType jType) {
+    this.javaType = jType;
+  }
+  
+  JavaType getJavaType() {
+    return javaType;
+  }
+  
+  void setCppType(CppType cppType) {
+    this.cppType = cppType;
+  }
+  
+  CppType getCppType() {
+    return cppType;
+  }
+  
+  void setCType(CType cType) {
+    this.cType = cType;
+  }
+  
+  CType getCType() {
+    return cType;
+  }
 }

src/java/org/apache/hadoop/record/compiler/JVector.java

@@ -18,141 +18,136 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JCompType.CCompType;
+import org.apache.hadoop.record.compiler.JCompType.CppCompType;
+
 /**
  *
  * @author Milind Bhandarkar
  */
 public class JVector extends JCompType {
-    
-    static private int level = 0;
-    
-    static private String getId(String id) { return id+getLevel(); }
-    
-    static private String getLevel() { return Integer.toString(level); }
-    
-    static private void incrLevel() { level++; }
-    
-    static private void decrLevel() { level--; }
-    
-    private JType mElement;
-    
-    /** Creates a new instance of JVector */
-    public JVector(JType t) {
-        super(" ::std::vector<"+t.getCppType()+">", "java.util.ArrayList", "Vector", "java.util.ArrayList");
-        mElement = t;
-    }
-    
-    public String getSignature() {
-        return "[" + mElement.getSignature() + "]";
+  
+  static private int level = 0;
+  
+  static private String getId(String id) { return id+getLevel(); }
+  
+  static private String getLevel() { return Integer.toString(level); }
+  
+  static private void incrLevel() { level++; }
+  
+  static private void decrLevel() { level--; }
+  
+  private JType type;
+  
+  class JavaVector extends JavaCompType {
+    
+    private JType.JavaType element;
+    
+    JavaVector(JType.JavaType t) {
+      super("java.util.ArrayList<"+t.getWrapperType()+">",
+          "Vector", "java.util.ArrayList<"+t.getWrapperType()+">");
+      element = t;
     }
     
-    public String genJavaCompareTo(String fname, String other) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("    {\n");
-      sb.append("      int "+getId("len1")+" = "+fname+".size();\n");
-      sb.append("      int "+getId("len2")+" = "+other+".size();\n");
-      sb.append("      for(int "+getId("vidx")+" = 0; "+getId("vidx")+"<"+
+    void genCompareTo(CodeBuffer cb, String fname, String other) {
+      cb.append("{\n");
+      cb.append("int "+getId("len1")+" = "+fname+".size();\n");
+      cb.append("int "+getId("len2")+" = "+other+".size();\n");
+      cb.append("for(int "+getId("vidx")+" = 0; "+getId("vidx")+"<"+
           getId("len1")+" && "+getId("vidx")+"<"+getId("len2")+"; "+
           getId("vidx")+"++) {\n");
-      sb.append("        "+mElement.getJavaWrapperType()+" "+getId("e1")+
-          " = ("+mElement.getJavaWrapperType()+") "+fname+
+      cb.append(element.getType()+" "+getId("e1")+
+          " = "+fname+
           ".get("+getId("vidx")+");\n");
-      sb.append("        "+mElement.getJavaWrapperType()+" "+getId("e2")+
-          " = ("+mElement.getJavaWrapperType()+") "+other+
+      cb.append(element.getType()+" "+getId("e2")+
+          " = "+other+
           ".get("+getId("vidx")+");\n");
-      sb.append(mElement.genJavaCompareToWrapper(getId("e1"), getId("e2")));
-      sb.append("         if (ret != 0) { return ret; }\n");
-      sb.append("      }\n");
-      sb.append("      ret = ("+getId("len1")+" - "+getId("len2")+");\n");
-      sb.append("    }\n");
-      return sb.toString();
+      element.genCompareTo(cb, getId("e1"), getId("e2"));
+      cb.append("if (ret != 0) { return ret; }\n");
+      cb.append("}\n");
+      cb.append("ret = ("+getId("len1")+" - "+getId("len2")+");\n");
+      cb.append("}\n");
     }
     
-    public String genJavaCompareToWrapper(String fname, String other) {
-      return genJavaCompareTo(fname, other);
-    }
-    
-    public String genJavaReadWrapper(String fname, String tag, boolean decl) {
-        StringBuffer ret = new StringBuffer("");
-        if (decl) {
-            ret.append("      java.util.ArrayList "+fname+";\n");
-        }
-        ret.append("    {\n");
-        incrLevel();
-        ret.append("      org.apache.hadoop.record.Index "+getId("vidx")+" = a_.startVector(\""+tag+"\");\n");
-        ret.append("      "+fname+"=new java.util.ArrayList();\n");
-        ret.append("      for (; !"+getId("vidx")+".done(); "+getId("vidx")+".incr()) {\n");
-        ret.append(mElement.genJavaReadWrapper(getId("e"), getId("e"), true));
-        ret.append("        "+fname+".add("+getId("e")+");\n");
-        ret.append("      }\n");
-        ret.append("    a_.endVector(\""+tag+"\");\n");
-        decrLevel();
-        ret.append("    }\n");
-        return ret.toString();
-    }
-    
-    public String genJavaReadMethod(String fname, String tag) {
-        return genJavaReadWrapper(fname, tag, false);
-    }
-    
-    public String genJavaWriteWrapper(String fname, String tag) {
-        StringBuffer ret = new StringBuffer("    {\n");
-        incrLevel();
-        ret.append("      a_.startVector("+fname+",\""+tag+"\");\n");
-        ret.append("      int "+getId("len")+" = "+fname+".size();\n");
-        ret.append("      for(int "+getId("vidx")+" = 0; "+getId("vidx")+"<"+getId("len")+"; "+getId("vidx")+"++) {\n");
-        ret.append("        "+mElement.getJavaWrapperType()+" "+getId("e")+" = ("+mElement.getJavaWrapperType()+") "+fname+".get("+getId("vidx")+");\n");
-        ret.append(mElement.genJavaWriteWrapper(getId("e"), getId("e")));
-        ret.append("      }\n");
-        ret.append("      a_.endVector("+fname+",\""+tag+"\");\n");
-        ret.append("    }\n");
-        decrLevel();
-        return ret.toString();
+    void genReadMethod(CodeBuffer cb, String fname, String tag, boolean decl) {
+      if (decl) {
+        cb.append(getType()+" "+fname+";\n");
+      }
+      cb.append("{\n");
+      incrLevel();
+      cb.append("org.apache.hadoop.record.Index "+getId("vidx")+" = a_.startVector(\""+tag+"\");\n");
+      cb.append(fname+"=new "+getType()+"();\n");
+      cb.append("for (; !"+getId("vidx")+".done(); "+getId("vidx")+".incr()) {\n");
+      element.genReadMethod(cb, getId("e"), getId("e"), true);
+      cb.append(fname+".add("+getId("e")+");\n");
+      cb.append("}\n");
+      cb.append("a_.endVector(\""+tag+"\");\n");
+      decrLevel();
+      cb.append("}\n");
     }
     
-    public String genJavaWriteMethod(String fname, String tag) {
-        return genJavaWriteWrapper(fname, tag);
+    void genWriteMethod(CodeBuffer cb, String fname, String tag) {
+      cb.append("{\n");
+      incrLevel();
+      cb.append("a_.startVector("+fname+",\""+tag+"\");\n");
+      cb.append("int "+getId("len")+" = "+fname+".size();\n");
+      cb.append("for(int "+getId("vidx")+" = 0; "+getId("vidx")+"<"+getId("len")+"; "+getId("vidx")+"++) {\n");
+      cb.append(element.getType()+" "+getId("e")+" = "+fname+".get("+getId("vidx")+");\n");
+      element.genWriteMethod(cb, getId("e"), getId("e"));
+      cb.append("}\n");
+      cb.append("a_.endVector("+fname+",\""+tag+"\");\n");
+      cb.append("}\n");
+      decrLevel();
     }
     
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
       incrLevel();
-      sb.append("           int "+getId("vi")+
-          " = WritableComparator.readVInt("+b+", "+s+");\n");
-      sb.append("           int "+getId("vz")+
-          " = WritableUtils.getVIntSize("+getId("vi")+");\n");
-      sb.append("           "+s+"+="+getId("vz")+"; "+l+"-="+getId("vz")+";\n");
-      sb.append("           for (int "+getId("vidx")+" = 0; "+getId("vidx")+
+      cb.append("int "+getId("vi")+
+          " = org.apache.hadoop.record.Utils.readVInt("+b+", "+s+");\n");
+      cb.append("int "+getId("vz")+
+          " = org.apache.hadoop.record.Utils.getVIntSize("+getId("vi")+");\n");
+      cb.append(s+"+="+getId("vz")+"; "+l+"-="+getId("vz")+";\n");
+      cb.append("for (int "+getId("vidx")+" = 0; "+getId("vidx")+
           " < "+getId("vi")+"; "+getId("vidx")+"++)");
-      sb.append(mElement.genJavaSlurpBytes(b,s,l));
+      element.genSlurpBytes(cb, b,s,l);
       decrLevel();
-      sb.append("        }\n");
-      return sb.toString();
+      cb.append("}\n");
     }
     
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
       incrLevel();
-      sb.append("           int "+getId("vi1")+
-          " = WritableComparator.readVInt(b1, s1);\n");
-      sb.append("           int "+getId("vi2")+
-          " = WritableComparator.readVInt(b2, s2);\n");
-      sb.append("           int "+getId("vz1")+
-          " = WritableUtils.getVIntSize("+getId("vi1")+");\n");
-      sb.append("           int "+getId("vz2")+
-          " = WritableUtils.getVIntSize("+getId("vi2")+");\n");
-      sb.append("           s1+="+getId("vz1")+"; s2+="+getId("vz2")+
+      cb.append("int "+getId("vi1")+
+          " = org.apache.hadoop.record.Utils.readVInt(b1, s1);\n");
+      cb.append("int "+getId("vi2")+
+          " = org.apache.hadoop.record.Utils.readVInt(b2, s2);\n");
+      cb.append("int "+getId("vz1")+
+          " = org.apache.hadoop.record.Utils.getVIntSize("+getId("vi1")+");\n");
+      cb.append("int "+getId("vz2")+
+          " = org.apache.hadoop.record.Utils.getVIntSize("+getId("vi2")+");\n");
+      cb.append("s1+="+getId("vz1")+"; s2+="+getId("vz2")+
           "; l1-="+getId("vz1")+"; l2-="+getId("vz2")+";\n");
-      sb.append("           for (int "+getId("vidx")+" = 0; "+getId("vidx")+
+      cb.append("for (int "+getId("vidx")+" = 0; "+getId("vidx")+
           " < "+getId("vi1")+" && "+getId("vidx")+" < "+getId("vi2")+
           "; "+getId("vidx")+"++)");
-      sb.append(mElement.genJavaCompareBytes());
-      sb.append("           if ("+getId("vi1")+" != "+getId("vi2")+
+      element.genCompareBytes(cb);
+      cb.append("if ("+getId("vi1")+" != "+getId("vi2")+
           ") { return ("+getId("vi1")+"<"+getId("vi2")+")?-1:0; }\n");
       decrLevel();
-      sb.append("        }\n");
-      return sb.toString();
+      cb.append("}\n");
     }
+  }
+  
+  /** Creates a new instance of JVector */
+  public JVector(JType t) {
+    type = t;
+    setJavaType(new JavaVector(t.getJavaType()));
+    setCppType(new CppCompType(" ::std::vector<"+t.getCppType().getType()+">"));
+    setCType(new CCompType());
+  }
+  
+  String getSignature() {
+    return "[" + type.getSignature() + "]";
+  }
 }

src/java/org/apache/hadoop/record/compiler/JavaGenerator.java

@@ -19,8 +19,6 @@
 package org.apache.hadoop.record.compiler;
 
 import java.util.ArrayList;
-import java.io.File;
-import java.io.FileWriter;
 import java.io.IOException;
 import java.util.Iterator;
 
@@ -29,34 +27,26 @@ import java.util.Iterator;
  *
  * @author Milind Bhandarkar
  */
-class JavaGenerator {
-    private String mName;
-    private String destDir;
-    private ArrayList mInclFiles;
-    private ArrayList mRecList;
-    
-    /** Creates a new instance of JavaGenerator
-     *
-     * @param name possibly full pathname to the file
-     * @param incl included files (as JFile)
-     * @param records List of records defined within this file
-     * @param destDir output directory
-     */
-    JavaGenerator(String name, ArrayList incl, ArrayList records, String destDir) {
-        mName = name;
-        mInclFiles = incl;
-        mRecList = records;
-        this.destDir = destDir;
-    }
-    
-    /**
-     * Generate Java code for records. This method is only a front-end to
-     * JRecord, since one file is generated for each record.
-     */
-    void genCode() throws IOException {
-        for (Iterator i = mRecList.iterator(); i.hasNext(); ) {
-            JRecord rec = (JRecord) i.next();
-            rec.genJavaCode(destDir);
-        }
+class JavaGenerator extends CodeGenerator {
+  
+  JavaGenerator() {
+  }
+  
+  /**
+   * Generate Java code for records. This method is only a front-end to
+   * JRecord, since one file is generated for each record.
+   *
+   * @param name possibly full pathname to the file
+   * @param ilist included files (as JFile)
+   * @param rlist List of records defined within this file
+   * @param destDir output directory
+   */
+  void genCode(String name, ArrayList<JFile> ilist,
+      ArrayList<JRecord> rlist, String destDir, ArrayList<String> options)
+      throws IOException {
+    for (Iterator<JRecord> iter = rlist.iterator(); iter.hasNext(); ) {
+      JRecord rec = iter.next();
+      rec.genJavaCode(destDir, options);
     }
+  }
 }

src/java/org/apache/hadoop/record/compiler/ant/RccTask.java

@@ -16,7 +16,7 @@
 package org.apache.hadoop.record.compiler.ant;
 
 import java.io.File;
-import java.util.Vector;
+import java.util.ArrayList;
 import org.apache.hadoop.record.compiler.generated.Rcc;
 import org.apache.tools.ant.BuildException;
 import org.apache.tools.ant.DirectoryScanner;
@@ -52,7 +52,7 @@ public class RccTask extends Task {
   private String language = "java";
   private File src;
   private File dest = new File(".");
-  private final Vector<FileSet> filesets = new Vector<FileSet>();
+  private final ArrayList<FileSet> filesets = new ArrayList<FileSet>();
   private boolean failOnError = true;
   
   /** Creates a new instance of RccTask */
@@ -111,7 +111,7 @@ public class RccTask extends Task {
     }
     Project myProject = getProject();
     for (int i = 0; i < filesets.size(); i++) {
-      FileSet fs = filesets.elementAt(i);
+      FileSet fs = filesets.get(i);
       DirectoryScanner ds = fs.getDirectoryScanner(myProject);
       File dir = fs.getDir(myProject);
       String[] srcs = ds.getIncludedFiles();

src/java/org/apache/hadoop/record/compiler/generated/Rcc.java

@@ -31,9 +31,10 @@ import java.io.IOException;
 public class Rcc implements RccConstants {
     private static String language = "java";
     private static String destDir = ".";
-    private static ArrayList recFiles = new ArrayList();
+    private static ArrayList<String> recFiles = new ArrayList<String>();
+    private static ArrayList<String> cmdargs = new ArrayList<String>();
     private static JFile curFile;
-    private static Hashtable recTab;
+    private static Hashtable<String,JRecord> recTab;
     private static String curDir = ".";
     private static String curFileName;
     private static String curModuleName;
@@ -52,26 +53,28 @@ public class Rcc implements RccConstants {
                 "--destdir".equalsIgnoreCase(args[i])) {
                 destDir = args[i+1];
                 i++;
+            } else if (args[i].startsWith("-")) {
+              String arg = args[i].substring(1);
+              if (arg.startsWith("-")) {
+                arg = arg.substring(1);
+              }
+              cmdargs.add(arg.toLowerCase());
             } else {
                 recFiles.add(args[i]);
             }
         }
-        if (!"c++".equals(language) && !"java".equals(language)) {
-            System.err.println("Cannot recognize language:" + language);
-            return 1;
-        }
         if (recFiles.size() == 0) {
             System.err.println("No record files specified. Exiting.");
             return 1;
         }
         for (int i=0; i<recFiles.size(); i++) {
-            curFileName = (String) recFiles.get(i);
+            curFileName = recFiles.get(i);
             File file = new File(curFileName);
             try {
                 FileReader reader = new FileReader(file);
                 Rcc parser = new Rcc(reader);
                 try {
-                    recTab = new Hashtable();
+                    recTab = new Hashtable<String,JRecord>();
                     curFile = parser.Input();
                 } catch (ParseException e) {
                     System.err.println(e.toString());
@@ -87,7 +90,7 @@ public class Rcc implements RccConstants {
                 return 1;
             }
             try {
-                int retCode = curFile.genCode(language, destDir);
+                int retCode = curFile.genCode(language, destDir, cmdargs);
                 if (retCode != 0) { return retCode; }
             } catch (IOException e) {
                 System.err.println(e.toString());
@@ -98,10 +101,10 @@ public class Rcc implements RccConstants {
     }
 
   final public JFile Input() throws ParseException {
-    ArrayList ilist = new ArrayList();
-    ArrayList rlist = new ArrayList();
+    ArrayList<JFile> ilist = new ArrayList<JFile>();
+    ArrayList<JRecord> rlist = new ArrayList<JRecord>();
     JFile i;
-    ArrayList l;
+    ArrayList<JRecord> l;
     label_1:
     while (true) {
       switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
@@ -170,9 +173,9 @@ public class Rcc implements RccConstants {
     throw new Error("Missing return statement in function");
   }
 
-  final public ArrayList Module() throws ParseException {
+  final public ArrayList<JRecord> Module() throws ParseException {
     String mName;
-    ArrayList rlist;
+    ArrayList<JRecord> rlist;
     jj_consume_token(MODULE_TKN);
     mName = ModuleName();
       curModuleName = mName;
@@ -206,8 +209,8 @@ public class Rcc implements RccConstants {
     throw new Error("Missing return statement in function");
   }
 
-  final public ArrayList RecordList() throws ParseException {
-    ArrayList rlist = new ArrayList();
+  final public ArrayList<JRecord> RecordList() throws ParseException {
+    ArrayList<JRecord> rlist = new ArrayList<JRecord>();
     JRecord r;
     label_3:
     while (true) {
@@ -228,9 +231,9 @@ public class Rcc implements RccConstants {
 
   final public JRecord Record() throws ParseException {
     String rname;
-    ArrayList flist = new ArrayList();
+    ArrayList<JField<JType>> flist = new ArrayList<JField<JType>>();
     Token t;
-    JField f;
+    JField<JType> f;
     jj_consume_token(RECORD_TKN);
     t = jj_consume_token(IDENT_TKN);
       rname = t.image;
@@ -267,12 +270,12 @@ public class Rcc implements RccConstants {
     throw new Error("Missing return statement in function");
   }
 
-  final public JField Field() throws ParseException {
+  final public JField<JType> Field() throws ParseException {
     JType jt;
     Token t;
     jt = Type();
     t = jj_consume_token(IDENT_TKN);
-      {if (true) return new JField(jt, t.image);}
+      {if (true) return new JField<JType>(t.image, jt);}
     throw new Error("Missing return statement in function");
   }
 
@@ -326,7 +329,7 @@ public class Rcc implements RccConstants {
         if (rname.indexOf('.', 0) < 0) {
             rname = curModuleName + "." + rname;
         }
-        JRecord r = (JRecord) recTab.get(rname);
+        JRecord r = recTab.get(rname);
         if (r == null) {
             System.out.println("Type " + rname + " not known. Exiting.");
             System.exit(1);

src/java/org/apache/hadoop/record/compiler/generated/rcc.jj

@@ -35,9 +35,10 @@ import java.io.IOException;
 public class Rcc {
     private static String language = "java";
     private static String destDir = ".";
-    private static ArrayList recFiles = new ArrayList();
+    private static ArrayList<String> recFiles = new ArrayList<String>();
+    private static ArrayList<String> cmdargs = new ArrayList<String>();
     private static JFile curFile;
-    private static Hashtable recTab;
+    private static Hashtable<String,JRecord> recTab;
     private static String curDir = ".";
     private static String curFileName;
     private static String curModuleName;
@@ -56,26 +57,28 @@ public class Rcc {
                 "--destdir".equalsIgnoreCase(args[i])) {
                 destDir = args[i+1];
                 i++;
+            } else if (args[i].startsWith("-")) {
+              String arg = args[i].substring(1);
+              if (arg.startsWith("-")) {
+                arg = arg.substring(1);
+              }
+              cmdargs.add(arg.toLowerCase());
             } else {
                 recFiles.add(args[i]);
             }
         }
-        if (!"c++".equals(language) && !"java".equals(language)) {
-            System.err.println("Cannot recognize language:" + language);
-            return 1;
-        }
         if (recFiles.size() == 0) {
             System.err.println("No record files specified. Exiting.");
             return 1;
         }
         for (int i=0; i<recFiles.size(); i++) {
-            curFileName = (String) recFiles.get(i);
+            curFileName = recFiles.get(i);
             File file = new File(curFileName);
             try {
                 FileReader reader = new FileReader(file);
                 Rcc parser = new Rcc(reader);
                 try {
-                    recTab = new Hashtable();
+                    recTab = new Hashtable<String,JRecord>();
                     curFile = parser.Input();
                 } catch (ParseException e) {
                     System.err.println(e.toString());
@@ -91,7 +94,7 @@ public class Rcc {
                 return 1;
             }
             try {
-                int retCode = curFile.genCode(language, destDir);
+                int retCode = curFile.genCode(language, destDir, cmdargs);
                 if (retCode != 0) { return retCode; }
             } catch (IOException e) {
                 System.err.println(e.toString());
@@ -170,10 +173,10 @@ TOKEN :
 
 JFile Input() :
 {
-    ArrayList ilist = new ArrayList();
-    ArrayList rlist = new ArrayList();
+    ArrayList<JFile> ilist = new ArrayList<JFile>();
+    ArrayList<JRecord> rlist = new ArrayList<JRecord>();
     JFile i;
-    ArrayList l;
+    ArrayList<JRecord> l;
 }
 {
     (
@@ -227,10 +230,10 @@ JFile Include() :
     }
 }
 
-ArrayList Module() :
+ArrayList<JRecord> Module() :
 {
     String mName;
-    ArrayList rlist;
+    ArrayList<JRecord> rlist;
 }
 {
     <MODULE_TKN>
@@ -258,9 +261,9 @@ String ModuleName() :
     { return name; }
 }
 
-ArrayList RecordList() :
+ArrayList<JRecord> RecordList() :
 {
-    ArrayList rlist = new ArrayList();
+    ArrayList<JRecord> rlist = new ArrayList<JRecord>();
     JRecord r;
 }
 {
@@ -274,9 +277,9 @@ ArrayList RecordList() :
 JRecord Record() :
 {
     String rname;
-    ArrayList flist = new ArrayList();
+    ArrayList<JField<JType>> flist = new ArrayList<JField<JType>>();
     Token t;
-    JField f;
+    JField<JType> f;
 }
 {
     <RECORD_TKN>
@@ -297,7 +300,7 @@ JRecord Record() :
     }
 }
 
-JField Field() :
+JField<JType> Field() :
 {
     JType jt;
     Token t;
@@ -305,7 +308,7 @@ JField Field() :
 {
     jt = Type()
     t = <IDENT_TKN>
-    { return new JField(jt, t.image); }
+    { return new JField<JType>(t.image, jt); }
 }
 
 JType Type() :
@@ -340,7 +343,7 @@ JType Type() :
         if (rname.indexOf('.', 0) < 0) {
             rname = curModuleName + "." + rname;
         }
-        JRecord r = (JRecord) recTab.get(rname);
+        JRecord r = recTab.get(rname);
         if (r == null) {
             System.out.println("Type " + rname + " not known. Exiting.");
             System.exit(1);

src/test/ddl/buffer.jr

@@ -1,6 +1,6 @@
-module org.apache.hadoop.record.test {
+module org.apache.hadoop.record {
     class RecBuffer {
-        buffer Data;
+        buffer data;
     }
 }
 

src/test/ddl/int.jr

@@ -1,6 +1,6 @@
-module org.apache.hadoop.record.test {
+module org.apache.hadoop.record {
     class RecInt {
-        int Data;
+        int data;
     }
 }
 

src/test/ddl/string.jr

@@ -1,6 +1,6 @@
-module org.apache.hadoop.record.test {
+module org.apache.hadoop.record {
     class RecString {
-        ustring Data;
+        ustring data;
     }
 }
 

src/test/ddl/test.jr

@@ -1,21 +1,21 @@
-module org.apache.hadoop.record.test {
+module org.apache.hadoop.record {
     class RecRecord0 {
-        ustring     StringVal;
+        ustring     stringVal;
     }
 
     class RecRecord1 {
-        boolean         BoolVal;
-        byte            ByteVal;
-        int             IntVal;
-        long            LongVal;
-        float           FloatVal; // testing inline comment
-        double          DoubleVal; /* testing comment */
-        ustring         StringVal; /* testing multi-line
+        boolean         boolVal;
+        byte            byteVal;
+        int             intVal;
+        long            longVal;
+        float           floatVal; // testing inline comment
+        double          doubleVal; /* testing comment */
+        ustring         stringVal; /* testing multi-line
                                     * comment */
-        buffer          BufferVal; // testing another // inline comment 
-        vector<ustring> VectorVal;
-        map<ustring,ustring>   MapVal;
-        RecRecord0      RecordVal;
+        buffer          bufferVal; // testing another // inline comment 
+        vector<ustring> vectorVal;
+        map<ustring, ustring>   mapVal;
+        RecRecord0      recordVal;
     }
 }
 

src/test/org/apache/hadoop/record/test/FromCpp.java → src/test/org/apache/hadoop/record/FromCpp.java

@@ -16,9 +16,8 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.record.test;
+package org.apache.hadoop.record;
 
-import org.apache.hadoop.record.RecordReader;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
@@ -26,8 +25,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.TreeMap;
 import junit.framework.*;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  *
@@ -56,10 +53,10 @@ public class FromCpp extends TestCase {
             r1.setDoubleVal(1.5234);
             r1.setIntVal(4567);
             r1.setLongVal(0x5a5a5a5a5a5aL);
-            r1.setStringVal(new Text("random text"));
-            r1.setBufferVal(new BytesWritable());
-            r1.setVectorVal(new ArrayList());
-            r1.setMapVal(new TreeMap());
+            r1.setStringVal("random text");
+            r1.setBufferVal(new Buffer());
+            r1.setVectorVal(new ArrayList<String>());
+            r1.setMapVal(new TreeMap<String,String>());
             FileInputStream istream = new FileInputStream(tmpfile);
             RecordReader in = new RecordReader(istream, "binary");
             RecRecord1 r2 = new RecRecord1();
@@ -82,10 +79,10 @@ public class FromCpp extends TestCase {
             r1.setDoubleVal(1.5234);
             r1.setIntVal(4567);
             r1.setLongVal(0x5a5a5a5a5a5aL);
-            r1.setStringVal(new Text("random text"));
-            r1.setBufferVal(new BytesWritable());
-            r1.setVectorVal(new ArrayList());
-            r1.setMapVal(new TreeMap());
+            r1.setStringVal("random text");
+            r1.setBufferVal(new Buffer());
+            r1.setVectorVal(new ArrayList<String>());
+            r1.setMapVal(new TreeMap<String,String>());
             FileInputStream istream = new FileInputStream(tmpfile);
             RecordReader in = new RecordReader(istream, "csv");
             RecRecord1 r2 = new RecRecord1();
@@ -108,10 +105,10 @@ public class FromCpp extends TestCase {
             r1.setDoubleVal(1.5234);
             r1.setIntVal(4567);
             r1.setLongVal(0x5a5a5a5a5a5aL);
-            r1.setStringVal(new Text("random text"));
-            r1.setBufferVal(new BytesWritable());
-            r1.setVectorVal(new ArrayList());
-            r1.setMapVal(new TreeMap());
+            r1.setStringVal("random text");
+            r1.setBufferVal(new Buffer());
+            r1.setVectorVal(new ArrayList<String>());
+            r1.setMapVal(new TreeMap<String,String>());
             FileInputStream istream = new FileInputStream(tmpfile);
             RecordReader in = new RecordReader(istream, "xml");
             RecRecord1 r2 = new RecRecord1();

src/test/org/apache/hadoop/record/TestBuffer.java

@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import junit.framework.*;
+
+/**
+ * A Unit test for Record I/O Buffer class
+ *
+ * @author milindb
+ */
+public class TestBuffer extends TestCase {
+  
+  public TestBuffer(String testName) {
+    super(testName);
+  }
+  
+  /**
+   * Test of set method, of class org.apache.hadoop.record.Buffer.
+   */
+  public void testSet() {
+    final byte[] bytes = new byte[10];
+    final Buffer instance = new Buffer();
+    
+    instance.set(bytes);
+    
+    assertEquals("set failed", bytes, instance.get());
+  }
+  
+  /**
+   * Test of copy method, of class org.apache.hadoop.record.Buffer.
+   */
+  public void testCopy() {
+    final byte[] bytes = new byte[10];
+    final int offset = 6;
+    final int length = 3;
+    for (int idx = 0; idx < 10; idx ++) {
+      bytes[idx] = (byte) idx;
+    }
+    final Buffer instance = new Buffer();
+    
+    instance.copy(bytes, offset, length);
+    
+    assertEquals("copy failed", 3, instance.getCapacity());
+    assertEquals("copy failed", 3, instance.get().length);
+    for (int idx = 0; idx < 3; idx++) {
+      assertEquals("Buffer content corrupted", idx+6, instance.get()[idx]);
+    }
+  }
+  
+  /**
+   * Test of getCount method, of class org.apache.hadoop.record.Buffer.
+   */
+  public void testGetCount() {
+    final Buffer instance = new Buffer();
+    
+    final int expResult = 0;
+    final int result = instance.getCount();
+    assertEquals("getSize failed", expResult, result);
+  }
+  
+  /**
+   * Test of getCapacity method, of class org.apache.hadoop.record.Buffer.
+   */
+  public void testGetCapacity() {
+    final Buffer instance = new Buffer();
+    
+    final int expResult = 0;
+    final int result = instance.getCapacity();
+    assertEquals("getCapacity failed", expResult, result);
+    
+    instance.setCapacity(100);
+    assertEquals("setCapacity failed", 100, instance.getCapacity());
+  }
+  
+  /**
+   * Test of truncate method, of class org.apache.hadoop.record.Buffer.
+   */
+  public void testTruncate() {
+    final Buffer instance = new Buffer();
+    instance.setCapacity(100);
+    assertEquals("setCapacity failed", 100, instance.getCapacity());
+    
+    instance.truncate();
+    assertEquals("truncate failed", 0, instance.getCapacity());
+  }
+  
+  /**
+   * Test of append method, of class org.apache.hadoop.record.Buffer.
+   */
+  public void testAppend() {
+    final byte[] bytes = new byte[100];
+    final int offset = 0;
+    final int length = 100;
+    for (int idx = 0; idx < 100; idx++) {
+      bytes[idx] = (byte) (100-idx);
+    }
+    
+    final Buffer instance = new Buffer();
+    
+    instance.append(bytes, offset, length);
+    
+    assertEquals("Buffer size mismatch", 100, instance.getCount());
+    
+    for (int idx = 0; idx < 100; idx++) {
+      assertEquals("Buffer contents corrupted", 100-idx, instance.get()[idx]);
+    }
+    
+  }
+}

src/test/org/apache/hadoop/record/test/TestRecordIO.java → src/test/org/apache/hadoop/record/TestRecordIO.java

@@ -16,19 +16,15 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.record.test;
+package org.apache.hadoop.record;
 
 import java.io.IOException;
 import junit.framework.*;
-import org.apache.hadoop.record.RecordWriter;
-import org.apache.hadoop.record.RecordReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.util.ArrayList;
 import java.util.TreeMap;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  *
@@ -57,14 +53,14 @@ public class TestRecordIO extends TestCase {
             r1.setByteVal((byte)0x66);
             r1.setFloatVal(3.145F);
             r1.setDoubleVal(1.5234);
-            r1.setIntVal(4567);
-            r1.setLongVal(0x5a5a5a5a5a5aL);
-            r1.setStringVal(new Text("random text"));
-            r1.setBufferVal(new BytesWritable());
-            r1.setVectorVal(new ArrayList());
-            r1.setMapVal(new TreeMap());
+            r1.setIntVal(-4567);
+            r1.setLongVal(-2367L);
+            r1.setStringVal("random text");
+            r1.setBufferVal(new Buffer());
+            r1.setVectorVal(new ArrayList<String>());
+            r1.setMapVal(new TreeMap<String,String>());
             RecRecord0 r0 = new RecRecord0();
-            r0.setStringVal(new Text("other random text"));
+            r0.setStringVal("other random text");
             r1.setRecordVal(r0);
             out.write(r1);
             ostream.close();
@@ -93,12 +89,12 @@ public class TestRecordIO extends TestCase {
             r1.setDoubleVal(1.5234);
             r1.setIntVal(4567);
             r1.setLongVal(0x5a5a5a5a5a5aL);
-            r1.setStringVal(new Text("random text"));
-            r1.setBufferVal(new BytesWritable());
-            r1.setVectorVal(new ArrayList());
-            r1.setMapVal(new TreeMap());
+            r1.setStringVal("random text");
+            r1.setBufferVal(new Buffer());
+            r1.setVectorVal(new ArrayList<String>());
+            r1.setMapVal(new TreeMap<String,String>());
             RecRecord0 r0 = new RecRecord0();
-            r0.setStringVal(new Text("other random text"));
+            r0.setStringVal("other random text");
             r1.setRecordVal(r0);
             out.write(r1);
             ostream.close();
@@ -124,12 +120,12 @@ public class TestRecordIO extends TestCase {
             r1.setDoubleVal(1.5234);
             r1.setIntVal(4567);
             r1.setLongVal(0x5a5a5a5a5a5aL);
-            r1.setStringVal(new Text("random text"));
-            r1.setBufferVal(new BytesWritable());
-            r1.setVectorVal(new ArrayList());
-            r1.setMapVal(new TreeMap());
+            r1.setStringVal("random text");
+            r1.setBufferVal(new Buffer());
+            r1.setVectorVal(new ArrayList<String>());
+            r1.setMapVal(new TreeMap<String,String>());
             RecRecord0 r0 = new RecRecord0();
-            r0.setStringVal(new Text("other random text"));
+            r0.setStringVal("other random text");
             r1.setRecordVal(r0);
             System.err.println("Illustrating toString bug"+r1.toString());
             System.err.println("Illustrating toString bug"+r1.toString());
@@ -152,12 +148,12 @@ public class TestRecordIO extends TestCase {
             r1.setDoubleVal(1.5234);
             r1.setIntVal(4567);
             r1.setLongVal(0x5a5a5a5a5a5aL);
-            r1.setStringVal(new Text("ran\002dom &lt; %text<&more"));
-            r1.setBufferVal(new BytesWritable());
-            r1.setVectorVal(new ArrayList());
-            r1.setMapVal(new TreeMap());
+            r1.setStringVal("ran\002dom &lt; %text<&more");
+            r1.setBufferVal(new Buffer());
+            r1.setVectorVal(new ArrayList<String>());
+            r1.setMapVal(new TreeMap<String,String>());
             RecRecord0 r0 = new RecRecord0();
-            r0.setStringVal(new Text("other %rando\007m &amp; >&more text"));
+            r0.setStringVal("other %rando\007m &amp; >&more text");
             r1.setRecordVal(r0);
             out.write(r1);
             ostream.close();
@@ -172,4 +168,30 @@ public class TestRecordIO extends TestCase {
             ex.printStackTrace();
         } 
     }
+    
+    public void testCloneable() {
+      RecRecord1 r1 = new RecRecord1();
+      r1.setBoolVal(true);
+      r1.setByteVal((byte)0x66);
+      r1.setFloatVal(3.145F);
+      r1.setDoubleVal(1.5234);
+      r1.setIntVal(-4567);
+      r1.setLongVal(-2367L);
+      r1.setStringVal("random text");
+      r1.setBufferVal(new Buffer());
+      r1.setVectorVal(new ArrayList<String>());
+      r1.setMapVal(new TreeMap<String,String>());
+      RecRecord0 r0 = new RecRecord0();
+      r0.setStringVal("other random text");
+      r1.setRecordVal(r0);
+      try {
+        RecRecord1 r2 = (RecRecord1) r1.clone();
+        assertTrue("Cloneable semantics violated. r1==r2", r1 != r2);
+        assertTrue("Cloneable semantics violated. r1.getClass() != r2.getClass()",
+            r1.getClass() == r2.getClass());
+        assertTrue("Cloneable semantics violated. !r2.equals(r1)", r2.equals(r1));
+      } catch (final CloneNotSupportedException ex) {
+        ex.printStackTrace();
+      }
+    }
 }

src/test/org/apache/hadoop/record/test/TestMapRed.java → src/test/org/apache/hadoop/record/TestRecordMR.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.record.test;
+package org.apache.hadoop.record;
 
 import org.apache.hadoop.mapred.*;
 import org.apache.hadoop.fs.*;
@@ -61,7 +61,7 @@ import java.util.*;
  * 7) A mapred job integrates all the count files into a single one.
  *
  **********************************************************/
-public class TestMapRed extends TestCase {
+public class TestRecordMR extends TestCase {
     /**
      * Modified to make it a junit test.
      * The RandomGen Job does the actual work of creating
@@ -97,7 +97,7 @@ public class TestMapRed extends TestCase {
 
             for (int i = 0; i < randomCount; i++) {
                 out.collect(new RecInt(Math.abs(r.nextInt())),
-                        new RecString(new Text(Integer.toString(randomVal))));
+                        new RecString(Integer.toString(randomVal)));
             }
         }
         public void close() {
@@ -116,9 +116,9 @@ public class TestMapRed extends TestCase {
                 throws IOException {
             int keyint = ((RecInt) key).getData();
             while (it.hasNext()) {
-                Text val = ((RecString) it.next()).getData();
-                out.collect(new RecInt(Integer.parseInt(val.toString())),
-                        new RecString(new Text("")));
+                String val = ((RecString) it.next()).getData();
+                out.collect(new RecInt(Integer.parseInt(val)),
+                        new RecString(""));
             }
         }
         public void close() {
@@ -147,8 +147,8 @@ public class TestMapRed extends TestCase {
 
         public void map(WritableComparable key, Writable val, OutputCollector out, Reporter reporter) throws IOException {
             int pos = ((RecInt) key).getData();
-            Text str = ((RecString) val).getData();
-            out.collect(new RecInt(pos), new RecString(new Text("1")));
+            String str = ((RecString) val).getData();
+            out.collect(new RecInt(pos), new RecString("1"));
         }
         public void close() {
         }
@@ -166,7 +166,7 @@ public class TestMapRed extends TestCase {
                 it.next();
                 count++;
             }
-            out.collect(new RecInt(keyint), new RecString(new Text(Integer.toString(count))));
+            out.collect(new RecInt(keyint), new RecString(Integer.toString(count)));
         }
         public void close() {
         }
@@ -186,8 +186,8 @@ public class TestMapRed extends TestCase {
 
         public void map(WritableComparable key, Writable val, OutputCollector out, Reporter reporter) throws IOException {
             int keyint = ((RecInt) key).getData();
-            Text valstr = ((RecString) val).getData();
-            out.collect(new RecInt(keyint), new RecInt(Integer.parseInt(valstr.toString())));
+            String valstr = ((RecString) val).getData();
+            out.collect(new RecInt(keyint), new RecInt(Integer.parseInt(valstr)));
         }
         public void close() {
         }
@@ -213,14 +213,6 @@ public class TestMapRed extends TestCase {
     private static Random r = new Random();
     private static Configuration conf = new Configuration();
 
-    /**
-       public TestMapRed(int range, int counts, Configuration conf) throws IOException {
-       this.range = range;
-       this.counts = counts;
-       this.conf = conf;
-       }
-    **/
-
     public void testMapred() throws Exception {
 	launch();
     }
@@ -295,7 +287,7 @@ public class TestMapRed extends TestCase {
         fs.delete(randomOuts);
 
 
-        JobConf genJob = new JobConf(conf,TestMapRed.class);
+        JobConf genJob = new JobConf(conf,TestRecordMR.class);
         genJob.setInputPath(randomIns);
         genJob.setInputKeyClass(RecInt.class);
         genJob.setInputValueClass(RecInt.class);
@@ -342,7 +334,7 @@ public class TestMapRed extends TestCase {
         int intermediateReduces = 10;
         Path intermediateOuts = new Path(testdir, "intermediateouts");
         fs.delete(intermediateOuts);
-        JobConf checkJob = new JobConf(conf,TestMapRed.class);
+        JobConf checkJob = new JobConf(conf,TestRecordMR.class);
         checkJob.setInputPath(randomOuts);
         checkJob.setInputKeyClass(RecInt.class);
         checkJob.setInputValueClass(RecString.class);
@@ -367,7 +359,7 @@ public class TestMapRed extends TestCase {
         //
         Path finalOuts = new Path(testdir, "finalouts");        
         fs.delete(finalOuts);
-        JobConf mergeJob = new JobConf(conf,TestMapRed.class);
+        JobConf mergeJob = new JobConf(conf,TestRecordMR.class);
         mergeJob.setInputPath(intermediateOuts);
         mergeJob.setInputKeyClass(RecInt.class);
         mergeJob.setInputValueClass(RecString.class);
@@ -447,7 +439,7 @@ public class TestMapRed extends TestCase {
      */
     public static void main(String[] argv) throws Exception {
         if (argv.length < 2) {
-            System.err.println("Usage: TestMapRed <range> <counts>");
+            System.err.println("Usage: TestRecordMR <range> <counts>");
             System.err.println();
             System.err.println("Note: a good test will have a <counts> value that is substantially larger than the <range>");
             return;

src/test/org/apache/hadoop/record/test/TestWritable.java → src/test/org/apache/hadoop/record/TestRecordWritable.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.record.test;
+package org.apache.hadoop.record;
 
 import java.io.*;
 import java.util.*;
@@ -34,7 +34,7 @@ import org.apache.hadoop.mapred.RecordReader;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 
-public class TestWritable extends TestCase {
+public class TestRecordWritable extends TestCase {
   private static final Log LOG = InputFormatBase.LOG;
 
   private static int MAX_LENGTH = 10000;
@@ -46,8 +46,11 @@ public class TestWritable extends TestCase {
     Path dir = new Path(System.getProperty("test.build.data",".") + "/mapred");
     Path file = new Path(dir, "test.seq");
     
-    // A reporter that does nothing
-    Reporter reporter = Reporter.NULL;
+    Reporter reporter = new Reporter() {
+        public void setStatus(String status) throws IOException {}
+        public void progress() throws IOException {}
+        public void incrCounter(Enum key, long amount) {}
+      };
     
     int seed = new Random().nextInt();
     //LOG.info("seed = "+seed);
@@ -72,7 +75,7 @@ public class TestWritable extends TestCase {
           byte[] data = new byte[random.nextInt(10)];
           random.nextBytes(data);
           RecBuffer value = new RecBuffer();
-          value.setData(new BytesWritable(data));
+          value.setData(new Buffer(data));
           writer.append(key, value);
         }
       } finally {
@@ -111,6 +114,6 @@ public class TestWritable extends TestCase {
   }
 
   public static void main(String[] args) throws Exception {
-    new TestWritable().testFormat();
+    new TestRecordWritable().testFormat();
   }
 }

src/test/org/apache/hadoop/record/test/ToCpp.java → src/test/org/apache/hadoop/record/ToCpp.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.record.test;
+package org.apache.hadoop.record;
 
 import java.io.IOException;
 import junit.framework.*;
@@ -26,8 +26,6 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.util.ArrayList;
 import java.util.TreeMap;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  *
@@ -58,10 +56,10 @@ public class ToCpp extends TestCase {
             r1.setDoubleVal(1.5234);
             r1.setIntVal(4567);
             r1.setLongVal(0x5a5a5a5a5a5aL);
-            r1.setStringVal(new Text("random text"));
-            r1.setBufferVal(new BytesWritable());
-            r1.setVectorVal(new ArrayList());
-            r1.setMapVal(new TreeMap());
+            r1.setStringVal("random text");
+            r1.setBufferVal(new Buffer());
+            r1.setVectorVal(new ArrayList<String>());
+            r1.setMapVal(new TreeMap<String,String>());
             out.write(r1);
             ostream.close();
         } catch (IOException ex) {
@@ -82,10 +80,10 @@ public class ToCpp extends TestCase {
             r1.setDoubleVal(1.5234);
             r1.setIntVal(4567);
             r1.setLongVal(0x5a5a5a5a5a5aL);
-            r1.setStringVal(new Text("random text"));
-            r1.setBufferVal(new BytesWritable());
-            r1.setVectorVal(new ArrayList());
-            r1.setMapVal(new TreeMap());
+            r1.setStringVal("random text");
+            r1.setBufferVal(new Buffer());
+            r1.setVectorVal(new ArrayList<String>());
+            r1.setMapVal(new TreeMap<String,String>());
             out.write(r1);
             ostream.close();
         } catch (IOException ex) {
@@ -106,10 +104,10 @@ public class ToCpp extends TestCase {
             r1.setDoubleVal(1.5234);
             r1.setIntVal(4567);
             r1.setLongVal(0x5a5a5a5a5a5aL);
-            r1.setStringVal(new Text("random text"));
-            r1.setBufferVal(new BytesWritable());
-            r1.setVectorVal(new ArrayList());
-            r1.setMapVal(new TreeMap());
+            r1.setStringVal("random text");
+            r1.setBufferVal(new Buffer());
+            r1.setVectorVal(new ArrayList<String>());
+            r1.setMapVal(new TreeMap<String,String>());
             out.write(r1);
             ostream.close();
         } catch (IOException ex) {