HADOOP-4138. Refactor the Hive SerDe library to better structure

the interfaces to the serializer and de-serializer.
(Zheng Shao via dhruba)



git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@696525 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -591,6 +591,10 @@ Trunk (unreleased changes)
 
     HADOOP-4197. Update DATA_TRANSFER_VERSION for HADOOP-3981. (szetszwo)
 
+    HADOOP-4138. Refactor the Hive SerDe library to better structure
+    the interfaces to the serializer and de-serializer.
+    (Zheng Shao via dhruba)
+
 Release 0.18.1 - 2008-09-17
 
   IMPROVEMENTS

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ByteStream.java

@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import java.io.*;
+
+/**
+ * Extensions to bytearrayinput/output streams
+ *
+ */
+public class ByteStream {
+  public static class Input extends ByteArrayInputStream {
+    public byte[] getData() { return buf; }
+    public int getCount() { return count;}
+    public void reset(byte [] argBuf, int argCount) {
+      buf = argBuf; mark = pos = 0; count = argCount;
+    }
+    public Input() {
+      super(new byte [1]);
+    }
+
+    public Input(byte[] buf) {
+      super(buf);
+    }
+    public Input(byte[] buf, int offset, int length) {
+      super(buf, offset, length);
+    }
+  }
+    
+  public static class Output extends ByteArrayOutputStream {
+    public byte[] getData() { return buf; }
+    public int getCount() { return count;}
+
+    public Output() { super(); }
+    public Output(int size) { super(size); }
+  }
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ByteStreamTypedSerDe.java

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import java.lang.reflect.Type;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+public abstract class ByteStreamTypedSerDe extends TypedSerDe {
+
+  protected ByteStream.Input bis;
+  protected ByteStream.Output bos;
+
+  public ByteStreamTypedSerDe(Type objectType) throws SerDeException {
+    super(objectType);
+    bos = new ByteStream.Output();
+    bis = new ByteStream.Input();
+  }
+
+  public Object deserialize(Writable field) throws SerDeException {
+    Object retObj = super.deserialize(field);
+    BytesWritable b = (BytesWritable)field;
+    bis.reset(b.get(), b.getSize());
+    return (retObj);
+  }
+
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ColumnSet.java

@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import java.util.ArrayList;
+
+public class ColumnSet {
+  public ArrayList<String> col;
+
+  public ColumnSet() {
+  }
+
+  public ColumnSet(ArrayList<String> col)
+  {
+    this();
+    this.col = col;
+  }
+
+  public String toString() {
+    return col.toString();
+  }
+  
+}
+

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/Deserializer.java

@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.conf.Configuration;
+import java.util.Properties;
+
+/**
+ * HiveDeserializer is used to deserialize the data from hadoop Writable to a 
+ * custom java object that can be of any type that the developer wants.
+ * 
+ * HiveDeserializer also provides the ObjectInspector which can be used to inspect 
+ * the internal structure of the object (that is returned by deserialize function).
+ *
+ */
+public interface Deserializer {
+
+  /**
+   * Initialize the HiveDeserializer.
+   * @param conf System properties
+   * @param tbl  table properties
+   * @throws SerDeException
+   */
+  public void initialize(Configuration conf, Properties tbl) throws SerDeException;
+  
+  /**
+   * Deserialize an object out of a Writable blob.
+   * In most cases, the return value of this function will be constant since the function
+   * will reuse the returned object.
+   * If the client wants to keep a copy of the object, the client needs to clone the
+   * returned value by calling ObjectInspectorUtils.getStandardObject().
+   * @param blob The Writable object containing a serialized object
+   * @return A Java object representing the contents in the blob.
+   */
+  public Object deserialize(Writable blob) throws SerDeException;
+
+  /**
+   * Get the object inspector that can be used to navigate through the internal
+   * structure of the Object returned from deserialize(...).
+   */
+  public ObjectInspector getObjectInspector() throws SerDeException;
+
+  public String getShortName();
+  
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java

@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.CharacterCodingException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.objectinspector.MetadataListStructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+
+public class MetadataTypedColumnsetSerDe implements SerDe {
+
+  public static final Log LOG = LogFactory.getLog(MetadataTypedColumnsetSerDe.class.getName());
+
+  public String getShortName() {
+    return shortName();
+  }
+
+
+  public static String shortName() {
+    return "simple_meta";
+  }
+
+  static {
+    StackTraceElement[] sTrace = new Exception().getStackTrace();
+    String className = sTrace[0].getClassName();
+    try {
+      SerDeUtils.registerSerDe(shortName(), Class.forName(className));
+      // For backward compatibility: this class replaces the following class.
+      SerDeUtils.registerSerDe("org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe", 
+          Class.forName(className));
+    } catch(Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+  
+  final public static String DefaultSeparator = "\001";
+
+  private String separator;
+  // constant for now, will make it configurable later.
+  private String nullString = "\\N"; 
+  private List<String> columnNames;
+  private ObjectInspector cachedObjectInspector;
+
+  public String toString() {
+    return "MetaDataTypedColumnsetSerDe[" + separator + "," + columnNames + "]";
+  }
+
+  public MetadataTypedColumnsetSerDe() throws SerDeException {
+    separator = DefaultSeparator;
+  }
+
+  public void initialize(Configuration job, Properties tbl) throws SerDeException {
+    separator = DefaultSeparator;
+    String alt_sep = tbl.getProperty(Constants.SERIALIZATION_FORMAT);
+    if(alt_sep != null && alt_sep.length() > 0) {
+      try {
+        byte b [] = new byte[1];
+        b[0] = Byte.valueOf(alt_sep).byteValue();
+        separator = new String(b);
+      } catch(NumberFormatException e) {
+        separator = alt_sep;
+      }
+    }
+    String columnProperty = tbl.getProperty("columns");
+    if (columnProperty == null || columnProperty.length() == 0) {
+      // Hack for tables with no columns
+      // Treat it as a table with a single column called "col" 
+      cachedObjectInspector = ObjectInspectorFactory.getReflectionObjectInspector(
+          ColumnSet.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+    } else {
+      columnNames = Arrays.asList(columnProperty.split(","));
+      cachedObjectInspector = MetadataListStructObjectInspector.getInstance(columnNames);
+    }
+    System.out.println(getClass().getName() + ": initialized with columnNames: " + columnNames );
+  }
+
+  public static Object deserialize(ColumnSet c, String row, String sep, String nullString) throws Exception {
+    if (c.col == null) {
+      c.col = new ArrayList<String>();
+    } else {
+      c.col.clear();
+    }
+    String [] l1 = row.split(sep, -1);
+
+    for(String s: l1) {
+      if (s.equals(nullString)) {
+        c.col.add(null);
+      } else {
+        c.col.add(s);
+      }
+    }
+    return (c);
+  }
+  
+  ColumnSet deserializeCache = new ColumnSet();
+  public Object deserialize(Writable field) throws SerDeException {
+    String row = null;
+    if (field instanceof BytesWritable) {
+      BytesWritable b = (BytesWritable)field;
+      try {
+        row = Text.decode(b.get(), 0, b.getSize());
+      } catch (CharacterCodingException e) {
+        throw new SerDeException(e);
+      }
+    } else if (field instanceof Text) {
+      row = field.toString();
+    }
+    try {
+      deserialize(deserializeCache, row, separator, nullString);
+      if (columnNames != null) {
+        assert(columnNames.size() == deserializeCache.col.size());
+      }
+      return deserializeCache;
+    } catch (ClassCastException e) {
+      throw new SerDeException( this.getClass().getName() + " expects Text or BytesWritable", e);
+    } catch (Exception e) {
+      throw new SerDeException(e);
+    }
+  }
+  
+  
+  public ObjectInspector getObjectInspector() throws SerDeException {
+    return cachedObjectInspector;
+  }
+
+  public Class<? extends Writable> getSerializedClass() {
+    return Text.class;
+  }
+  
+  Text serializeCache = new Text();
+  public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
+
+    if (objInspector.getCategory() != Category.STRUCT) {
+      throw new SerDeException(getClass().toString() 
+          + " can only serialize struct types, but we got: " + objInspector.getTypeName());
+    }
+    StructObjectInspector soi = (StructObjectInspector) objInspector;
+    List<? extends StructField> fields = soi.getAllStructFieldRefs();
+    
+    StringBuilder sb = new StringBuilder();
+    for(int i=0; i<fields.size(); i++) {
+      if (i>0) sb.append(separator);
+      Object column = soi.getStructFieldData(obj, fields.get(i));
+      if (fields.get(i).getFieldObjectInspector().getCategory() == Category.PRIMITIVE) {
+        // For primitive object, serialize to plain string
+        sb.append(column == null ? nullString : column.toString());
+      } else {
+        // For complex object, serialize to JSON format
+        sb.append(SerDeUtils.getJSONString(column, fields.get(i).getFieldObjectInspector()));
+      }
+    }
+    serializeCache.set(sb.toString());
+    return serializeCache;
+  }
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/SerDe.java

@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+/**
+ * A union of HiveDeserializer and HiveSerializer interface.
+ * 
+ * If a developer wants his hive table to be read-only, then he just want to 
+ * return 
+ * 
+ * both readable and writable, then
+ *   
+ *
+ */
+public interface SerDe extends Deserializer, Serializer {
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/SerDeException.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+/**
+ * Generic exception class for SerDes
+ *
+ */
+
+public class SerDeException extends Exception {
+  private static final long serialVersionUID = 1L;
+
+  public SerDeException() {
+    super();
+  }
+
+  public SerDeException(String message) {
+    super(message);
+  }
+
+  public SerDeException(Throwable cause) {
+    super(cause);
+  }
+
+  public SerDeException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}
+

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java

@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import java.util.*;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
+public class SerDeUtils {
+
+
+  public static final char QUOTE = '"';
+  public static final char COLON = ':';
+  public static final char COMMA = ',';
+  public static final String LBRACKET = "[";
+  public static final String RBRACKET = "]";
+  public static final String LBRACE = "{";
+  public static final String RBRACE = "}";
+
+  private static HashMap<String, Class<?>> serdes = new HashMap<String, Class<?>> ();
+
+  public static void registerSerDe(String name, Class<?> serde) {
+    if(serdes.containsKey(name)) {
+      throw new RuntimeException("double registering serde " + name);
+    }
+    serdes.put(name, serde);
+  }
+
+  public static Deserializer lookupDeserializer(String name) throws SerDeException {
+    Class<?> c;
+    if(serdes.containsKey(name)) {
+        c = serdes.get(name);
+    } else {
+      try {
+        c = Class.forName(name);
+      } catch(ClassNotFoundException e) {
+        throw new SerDeException("SerDe " + name + " does not exist");
+      }
+    }
+    try {
+      return (Deserializer)c.newInstance();
+    } catch(Exception e) {
+      throw new SerDeException(e);
+    }
+  }
+
+
+  private static boolean initCoreSerDes = registerCoreSerDes();
+  
+  protected static boolean registerCoreSerDes() {
+    // Eagerly load SerDes so they will register their symbolic names even on Lazy Loading JVMs
+    try {
+      // loading these classes will automatically register the short names
+      Class.forName(org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
+      Class.forName(org.apache.hadoop.hive.serde2.ThriftDeserializer.class.getName());
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException("IMPOSSIBLE Exception: Unable to initialize core serdes", e);
+    }
+    return true;
+  }
+
+  public static String escapeString(String str) {
+    int length = str.length();
+    StringBuilder escape = new StringBuilder(length + 16);
+
+    for (int i = 0; i < length; ++i) {
+      char c = str.charAt(i);
+      switch (c) {
+      case '"':
+      case '\\':
+        escape.append('\\');
+        escape.append(c);
+        break;
+      case '\b':
+        escape.append('\\');
+        escape.append('b');
+        break;
+      case '\f':
+        escape.append('\\');
+        escape.append('f');
+        break;
+      case '\n':
+        escape.append('\\');
+        escape.append('n');
+        break;
+      case '\r':
+        escape.append('\\');
+        escape.append('r');
+        break;
+      case '\t':
+        escape.append('\\');
+        escape.append('t');
+        break;
+      default:
+        // Control characeters! According to JSON RFC u0020
+        if (c < ' ') {
+          String hex = Integer.toHexString(c);
+          escape.append('\\');
+          escape.append('u');
+          for (int j = 4; j > hex.length(); --j) {
+            escape.append('0');
+          }
+          escape.append(hex);
+        } else {
+          escape.append(c);
+        }
+        break;
+      }
+    }
+    return (escape.toString());
+  }
+
+
+  public static String lightEscapeString(String str) {
+    int length = str.length();
+    StringBuilder escape = new StringBuilder(length + 16);
+
+    for (int i = 0; i < length; ++i) {
+      char c = str.charAt(i);
+      switch (c) {
+      case '\n':
+        escape.append('\\');
+        escape.append('n');
+        break;
+      case '\r':
+        escape.append('\\');
+        escape.append('r');
+        break;
+      case '\t':
+        escape.append('\\');
+        escape.append('t');
+        break;
+      default:
+        escape.append(c);
+        break;
+      }
+    }
+    return (escape.toString());
+  }
+
+  public static String getJSONString(Object o, ObjectInspector oi) {
+    StringBuilder sb = new StringBuilder();
+    buildJSONString(sb, o, oi);
+    return sb.toString();
+  }
+
+  
+  static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi) {
+
+    switch(oi.getCategory()) {
+      case PRIMITIVE: {
+        if (o == null) {
+          sb.append("\\N");
+        } else if (o instanceof String) {
+          sb.append(QUOTE);
+          sb.append(escapeString((String)o));
+          sb.append(QUOTE);
+        } else if (o instanceof Boolean) {
+          sb.append(((Boolean)o).booleanValue() ? "True" : "False");
+        } else {
+          // it's a number - so doesn't need to be escaped.
+          sb.append(o.toString());
+        }
+        break;
+      }
+      case LIST: {
+        sb.append(LBRACKET);
+        ListObjectInspector loi = (ListObjectInspector)oi;
+        ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
+        List<?> olist = loi.getList(o);
+        for (int i=0; i<olist.size(); i++) {
+          if (i>0) sb.append(COMMA);
+          buildJSONString(sb, olist.get(i), listElementObjectInspector);
+        }
+        sb.append(RBRACKET);
+        break;
+      }
+      case MAP: {
+        sb.append(LBRACE);
+        MapObjectInspector moi = (MapObjectInspector)oi;
+        ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
+        ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
+        Map<?,?> omap = moi.getMap(o);
+        boolean first = true;
+        for(Object entry : omap.entrySet()) {
+          if (first) {
+            first = false;
+          } else {
+            sb.append(COMMA);
+          }
+          Map.Entry<?,?> e = (Map.Entry<?,?>)entry;
+          buildJSONString(sb, e.getKey(), mapKeyObjectInspector);
+          sb.append(COLON);
+          buildJSONString(sb, e.getValue(), mapValueObjectInspector);
+        }
+        sb.append(RBRACE);
+        break;
+      }
+      case STRUCT: {
+        sb.append(LBRACE);
+        StructObjectInspector soi = (StructObjectInspector)oi;
+        List<? extends StructField> structFields = soi.getAllStructFieldRefs();
+        for(int i=0; i<structFields.size(); i++) {
+          if (i>0) {
+            sb.append(COMMA);
+          }
+          sb.append(QUOTE);
+          sb.append(structFields.get(i).getFieldName());
+          sb.append(QUOTE);
+          sb.append(COLON);
+          buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), 
+              structFields.get(i).getFieldObjectInspector());          
+        }
+        sb.append(RBRACE);
+        break;
+      }
+      default:
+        throw new RuntimeException("Unknown type in ObjectInspector!");
+    };
+    
+  }  
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java

@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.conf.Configuration;
+import java.util.Properties;
+
+/**
+ * HiveSerializer is used to serialize data to a Hadoop Writable object.
+ * The serialize 
+ * In addition to the interface below, all implementations are assume to have a ctor
+ * that takes a single 'Table' object as argument.
+ *
+ */
+public interface Serializer {
+
+  /**
+   * Initialize the HiveSerializer.
+   * @param conf System properties
+   * @param tbl  table properties
+   * @throws SerDeException
+   */
+  public void initialize(Configuration conf, Properties tbl) throws SerDeException;
+  
+  /**
+   * Returns the Writable class that would be returned by the serialize method.
+   * This is used to initialize SequenceFile header.
+   */
+  public Class<? extends Writable> getSerializedClass();
+  /**
+   * Serialize an object by navigating inside the Object with the ObjectInspector.
+   * In most cases, the return value of this function will be constant since the function
+   * will reuse the Writable object.
+   * If the client wants to keep a copy of the Writable, the client needs to clone the
+   * returned value.
+   */
+  public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException;
+
+  public String getShortName();
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/TReflectionUtils.java

@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import com.facebook.thrift.protocol.TProtocolFactory;
+
+
+public class TReflectionUtils {
+  public static final String thriftReaderFname = "read";
+  public static final String thriftWriterFname = "write";
+
+  public static final Class<?> [] thriftRWParams;
+  static {
+    try {
+      thriftRWParams = new Class [] {
+      Class.forName("com.facebook.thrift.protocol.TProtocol")
+      };
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public static TProtocolFactory getProtocolFactoryByName(String protocolName)
+    throws Exception {
+    Class<?> protoClass = Class.forName(protocolName + "$Factory");
+    return ((TProtocolFactory)protoClass.newInstance());
+  }
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ThriftByteStreamTypedSerDe.java

@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import java.lang.reflect.Type;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Writable;
+
+import com.facebook.thrift.TBase;
+import com.facebook.thrift.protocol.TProtocol;
+import com.facebook.thrift.protocol.TProtocolFactory;
+import com.facebook.thrift.transport.TIOStreamTransport;
+
+public class ThriftByteStreamTypedSerDe extends ByteStreamTypedSerDe {
+
+  protected TIOStreamTransport outTransport, inTransport;
+  protected TProtocol outProtocol, inProtocol;
+
+  private void init(TProtocolFactory inFactory, TProtocolFactory outFactory) throws Exception {
+    outTransport = new TIOStreamTransport(bos);
+    inTransport = new TIOStreamTransport(bis);
+    outProtocol = outFactory.getProtocol(outTransport);
+    inProtocol = inFactory.getProtocol(inTransport);
+  }
+
+  public void initialize(Configuration job, Properties tbl) throws SerDeException {
+    throw new SerDeException("ThriftByteStreamTypedSerDe is still semi-abstract");
+  }
+
+  public static String shortName() {
+    return "thriftbytestream";
+  }
+
+  public  String getShortName() {
+    return shortName();
+  }
+
+
+  public ThriftByteStreamTypedSerDe(Type objectType, TProtocolFactory inFactory,
+                                    TProtocolFactory outFactory) throws SerDeException {
+    super(objectType);
+    try {
+      init(inFactory, outFactory);
+    } catch (Exception e) {
+      throw new SerDeException(e);
+    }
+  }
+
+  protected ObjectInspectorFactory.ObjectInspectorOptions getObjectInspectorOptions() {
+    return ObjectInspectorFactory.ObjectInspectorOptions.THRIFT;
+  }
+  
+  public Object deserialize(Writable field) throws SerDeException {
+    Object obj = super.deserialize(field);
+    try {
+      ((TBase)obj).read(inProtocol);
+    } catch (Exception e) {
+      throw new SerDeException(e);
+    }
+    return obj;
+  }
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ThriftDeserializer.java

@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Writable;
+import com.facebook.thrift.protocol.TProtocolFactory;
+
+public class ThriftDeserializer implements Deserializer {
+
+  public static String shortName() {
+    return "thrift";
+  }
+
+  public  String getShortName() {
+    return shortName();
+  }
+
+  static {
+    StackTraceElement[] sTrace = new Exception().getStackTrace();
+    String className = sTrace[0].getClassName();
+    try {
+      SerDeUtils.registerSerDe(shortName(), Class.forName(className));
+      // For backward compatibility: this class replaces the following class.
+      SerDeUtils.registerSerDe("org.apache.hadoop.hive.serde.thrift.ThriftSerDe", Class.forName(className));
+    } catch(Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+  
+  private ThriftByteStreamTypedSerDe tsd;
+
+  public ThriftDeserializer() { }
+  
+  public void initialize(Configuration job, Properties tbl) throws SerDeException {
+    try {
+      // both the classname and the protocol name are Table properties
+      // the only hardwired assumption is that records are fixed on a
+      // per Table basis
+
+      String className = tbl.getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS);
+      Class<?> recordClass = Class.forName(className);
+
+      String protoName = tbl.getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT);
+      if (protoName == null) {
+        protoName = "TBinaryProtocol";
+      }
+
+      TProtocolFactory tp = TReflectionUtils.getProtocolFactoryByName(protoName);
+      tsd = new ThriftByteStreamTypedSerDe(recordClass, tp, tp);
+      
+    } catch (Exception e) {
+      throw new SerDeException(e);
+    }
+  }
+
+  public Object deserialize(Writable field) throws SerDeException {
+    return tsd.deserialize(field);
+  }
+  
+  public ObjectInspector getObjectInspector() throws SerDeException {
+    return tsd.getObjectInspector();
+  }
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/TypedSerDe.java

@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ReflectionUtils;
+
+public abstract class TypedSerDe implements SerDe {
+
+  protected Type objectType;
+  protected Class<?> objectClass;
+
+  public TypedSerDe(Type objectType) throws SerDeException {
+    this.objectType = objectType;
+    if (objectType instanceof Class) {
+      objectClass = (Class<?>)objectType;
+    } else if (objectType instanceof ParameterizedType) {
+      objectClass = (Class<?>)(((ParameterizedType)objectType).getRawType());
+    } else {
+      throw new SerDeException("Cannot create TypedSerDe with type " + objectType);
+    }
+  }
+
+  protected Object deserializeCache;
+  public Object deserialize(Writable blob) throws SerDeException {
+    if (deserializeCache == null) {
+      return ReflectionUtils.newInstance(objectClass, null);
+    } else {
+      assert(deserializeCache.getClass().equals(objectClass));
+      return deserializeCache;
+    }
+  }
+
+  public ObjectInspector getObjectInspector() throws SerDeException {
+    return ObjectInspectorFactory.getReflectionObjectInspector(objectType,
+        getObjectInspectorOptions());
+  }
+
+  protected ObjectInspectorFactory.ObjectInspectorOptions getObjectInspectorOptions() {
+    return ObjectInspectorFactory.ObjectInspectorOptions.JAVA;
+  }
+  
+  public void initialize(Configuration job, Properties tbl)
+      throws SerDeException {
+    // do nothing
+  }
+
+  public Class<? extends Writable> getSerializedClass() {
+    return BytesWritable.class;
+  }
+  public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
+    throw new RuntimeException("not supported");
+  }
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/InspectableObject.java

@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+/**
+ * Simple wrapper of object with ObjectInspector.
+ * 
+ * TODO: we need to redefine the hashCode and equals methods, so that
+ * it can be put into a HashMap as a key.
+ * 
+ * This class also serves as a facility for a function that returns
+ * both an object and an ObjectInspector.
+ */
+public class InspectableObject {
+
+  public Object o;
+  public ObjectInspector oi;
+  
+  public InspectableObject() {
+    this(null,null);
+  }
+  public InspectableObject(Object o, ObjectInspector oi) {
+    this.o = o;
+    this.oi = oi;
+  }
+  
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.List;
+
+
+public interface ListObjectInspector extends ObjectInspector {
+
+  // ** Methods that does not need a data object **
+  public ObjectInspector getListElementObjectInspector();
+
+  // ** Methods that need a data object **
+  /** returns null for null list, out-of-the-range index.
+   */
+  public Object getListElement(Object data, int index);
+
+  /** returns -1 for data = null.
+   */
+  public int getListLength(Object data);
+  
+  /** returns null for data = null.
+   */
+  public List<?> getList(Object data);
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java

@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.Map;
+
+
+public interface MapObjectInspector extends ObjectInspector {
+
+  // ** Methods that does not need a data object **
+  // Map Type
+  public ObjectInspector getMapKeyObjectInspector();
+
+  public ObjectInspector getMapValueObjectInspector();
+
+  // ** Methods that need a data object **
+  // In this function, key has to be of the same structure as the Map expects.
+  // Most cases key will be primitive type, so it's OK.
+  // In rare cases that key is not primitive, the user is responsible for defining 
+  // the hashCode() and equals() methods of the key class.
+  public Object getMapValueElement(Object data, Object key);
+
+  /** returns null for data = null.
+   */
+  public Map<?,?> getMap(Object data);
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java

@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hive.serde2.ColumnSet;
+
+/**
+ * StructObjectInspector works on struct data that is stored as a Java List or Java Array object.
+ * Basically, the fields are stored sequentially in the List object.
+ * 
+ * The names of the struct fields and the internal structure of the struct fields are specified in 
+ * the ctor of the StructObjectInspector.
+ * 
+ */
+public class MetadataListStructObjectInspector extends StandardStructObjectInspector {
+
+  static HashMap<List<String>, MetadataListStructObjectInspector> cached
+     = new HashMap<List<String>, MetadataListStructObjectInspector>();
+  public static MetadataListStructObjectInspector getInstance(int fields) {
+    return getInstance(ObjectInspectorUtils.getIntegerArray(fields));
+  }
+  public static MetadataListStructObjectInspector getInstance(List<String> columnNames) {
+    MetadataListStructObjectInspector result = cached.get(columnNames);
+    if (result == null) {
+      result = new MetadataListStructObjectInspector(columnNames);
+      cached.put(columnNames, result);
+    }
+    return result;
+  }
+
+  static ArrayList<ObjectInspector> getFieldObjectInspectors(int fields) {
+    ArrayList<ObjectInspector> r = new ArrayList<ObjectInspector>(fields);
+    for(int i=0; i<fields; i++) {
+      r.add(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class));
+    }
+    return r;
+  }
+  
+  MetadataListStructObjectInspector(List<String> columnNames) {
+    super(columnNames, getFieldObjectInspectors(columnNames.size()));
+  }
+  
+  // Get col object out
+  public Object getStructFieldData(Object data, StructField fieldRef) {
+    if (data instanceof ColumnSet) {
+      data = ((ColumnSet)data).col;
+    }
+    return super.getStructFieldData(data, fieldRef);
+  }
+  // Get col object out
+  public List<Object> getStructFieldsDataAsList(Object data) {
+    if (data instanceof ColumnSet) {
+      data = ((ColumnSet)data).col;
+    }
+    return super.getStructFieldsDataAsList(data);
+  }
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java

@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+/**
+ * ObjectInspector helps us to look into the internal structure of a complex
+ * object.
+ *
+ * A (probably configured) ObjectInspector instance stands for a specific type
+ * and a specific way to store the data of that type in the memory.
+ * 
+ * For native java Object, we can directly access the internal structure through 
+ * member fields and methods.  ObjectInspector is a way to delegate that functionality
+ * away from the Object, so that we have more control on the behavior of those actions.
+ * 
+ * An efficient implementation of ObjectInspector should rely on factory, so that we can 
+ * make sure the same ObjectInspector only has one instance.  That also makes sure
+ * hashCode() and equals() methods of java.lang.Object directly works for ObjectInspector
+ * as well.
+ */
+public interface ObjectInspector {
+
+  public static enum Category {
+    PRIMITIVE, LIST, MAP, STRUCT
+  };
+
+  /**
+   * Returns the name of the data type that is inspected by this ObjectInspector.
+   * This is used to display the type information to the user.
+   * 
+   * For primitive types, the type name is standardized.
+   * For other types, the type name can be something like "list<int>", "map<int,string>",
+   * java class names, or user-defined type names similar to typedef. 
+   */
+  public String getTypeName();
+  
+  /**
+   * An ObjectInspector must inherit from one of the following interfaces
+   * if getCategory() returns:
+   * PRIMITIVE:  PrimitiveObjectInspector 
+   * LIST:       ListObjectInspector 
+   * MAP:        MapObjectInspector 
+   * STRUCT:     StructObjectInspector 
+   */
+  public Category getCategory();
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java

@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.GenericArrayType;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * ObjectInspectorFactory is the primary way to create new ObjectInspector instances.
+ * 
+ * SerDe classes should call the static functions in this library to create an ObjectInspector
+ * to return to the caller of SerDe2.getObjectInspector(). 
+ */
+public class ObjectInspectorFactory {
+
+
+  /**
+   * ObjectInspectorOptions describes what ObjectInspector to use. 
+   * JAVA is to use pure JAVA reflection. THRIFT is to use JAVA reflection and filter out __isset fields.
+   * New ObjectInspectorOptions can be added here when available.
+   * 
+   * We choose to use a single HashMap objectInspectorCache to cache all situations for efficiency and code 
+   * simplicity.  And we don't expect a case that a user need to create 2 or more different types of 
+   * ObjectInspectors for the same Java type.
+   */
+  public enum ObjectInspectorOptions {
+    JAVA,
+    THRIFT
+  };
+  
+  private static HashMap<Type, ObjectInspector> objectInspectorCache = new HashMap<Type, ObjectInspector>();
+  
+  public static ObjectInspector getReflectionObjectInspector(Type t, ObjectInspectorOptions options) {
+    ObjectInspector oi = objectInspectorCache.get(t);
+    if (oi == null) {
+      oi = getReflectionObjectInspectorNoCache(t, options);
+      objectInspectorCache.put(t, oi);
+    }
+    if ((options.equals(ObjectInspectorOptions.JAVA) && oi.getClass().equals(ThriftStructObjectInspector.class))
+        || (options.equals(ObjectInspectorOptions.THRIFT) && oi.getClass().equals(ReflectionStructObjectInspector.class))) {
+      throw new RuntimeException("Cannot call getObjectInspectorByReflection with both JAVA and THRIFT !");
+    }
+    return oi;
+  }
+  
+  private static ObjectInspector getReflectionObjectInspectorNoCache(Type t, ObjectInspectorOptions options) {
+    if (t instanceof GenericArrayType) {
+      GenericArrayType at = (GenericArrayType)t;
+      return getStandardListObjectInspector(
+          getReflectionObjectInspector(at.getGenericComponentType(), options));
+    }
+
+    if (t instanceof ParameterizedType) {
+      ParameterizedType pt = (ParameterizedType)t;
+      // List?
+      if (List.class.isAssignableFrom((Class<?>)pt.getRawType())) {
+        return getStandardListObjectInspector(
+            getReflectionObjectInspector(pt.getActualTypeArguments()[0], options));
+      }
+      // Map?
+      if (Map.class.isAssignableFrom((Class<?>)pt.getRawType())) {
+        return getStandardMapObjectInspector(
+            getReflectionObjectInspector(pt.getActualTypeArguments()[0], options),
+            getReflectionObjectInspector(pt.getActualTypeArguments()[1], options));
+      }
+      // Otherwise convert t to RawType so we will fall into the following if block.
+      t = pt.getRawType();
+    }
+    
+    // Must be a class.
+    if (!(t instanceof Class)) {
+      throw new RuntimeException(ObjectInspectorFactory.class.getName() + ": internal error."); 
+    }
+    Class<?> c = (Class<?>)t;
+    
+    // Primitive?
+    if (ObjectInspectorUtils.isPrimitiveClass(c)) {
+      return getStandardPrimitiveObjectInspector(c);
+    }
+    
+    // Must be struct because List and Map need to be ParameterizedType
+    assert(!List.class.isAssignableFrom(c));
+    assert(!Map.class.isAssignableFrom(c));
+    
+    // Create StructObjectInspector
+    ReflectionStructObjectInspector oi;
+    switch(options) {
+    case JAVA: 
+      oi = new ReflectionStructObjectInspector();
+      break;
+    case THRIFT: 
+      oi = new ThriftStructObjectInspector();
+      break;
+    default:
+      throw new RuntimeException(ObjectInspectorFactory.class.getName() + ": internal error."); 
+    }
+    // put it into the cache BEFORE it is initialized to make sure we can catch recursive types. 
+    objectInspectorCache.put(t, oi);
+    Field[] fields = c.getDeclaredFields();
+    ArrayList<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>(fields.length);
+    for(int i=0; i<fields.length; i++) {
+      if (!oi.shouldIgnoreField(fields[i].getName())) {
+        structFieldObjectInspectors.add(getReflectionObjectInspector(fields[i].getGenericType(), options));
+      }
+    }
+    oi.init(c, structFieldObjectInspectors);
+    return oi;
+  }
+  
+  
+  private static HashMap<Class<?>, StandardPrimitiveObjectInspector> cachedStandardPrimitiveInspectorCache = new HashMap<Class<?>, StandardPrimitiveObjectInspector>();
+  public static StandardPrimitiveObjectInspector getStandardPrimitiveObjectInspector(Class<?> c) {
+    c = ObjectInspectorUtils.generalizePrimitive(c);
+    StandardPrimitiveObjectInspector result = cachedStandardPrimitiveInspectorCache.get(c);
+    if (result == null) {
+      result = new StandardPrimitiveObjectInspector(c);
+      cachedStandardPrimitiveInspectorCache.put(c, result);
+    }
+    return result;
+  }
+  
+  static HashMap<ObjectInspector, StandardListObjectInspector> cachedStandardListObjectInspector =
+    new HashMap<ObjectInspector, StandardListObjectInspector>(); 
+  public static StandardListObjectInspector getStandardListObjectInspector(ObjectInspector listElementObjectInspector) {
+    StandardListObjectInspector result = cachedStandardListObjectInspector.get(listElementObjectInspector);
+    if (result == null) {
+      result = new StandardListObjectInspector(listElementObjectInspector);
+      cachedStandardListObjectInspector.put(listElementObjectInspector, result);
+    }
+    return result;
+  }
+
+  static HashMap<List<ObjectInspector>, StandardMapObjectInspector> cachedStandardMapObjectInspector =
+    new HashMap<List<ObjectInspector>, StandardMapObjectInspector>(); 
+  public static StandardMapObjectInspector getStandardMapObjectInspector(ObjectInspector mapKeyObjectInspector, ObjectInspector mapValueObjectInspector) {
+    ArrayList<ObjectInspector> signature = new ArrayList<ObjectInspector>(2);
+    signature.add(mapKeyObjectInspector);
+    signature.add(mapValueObjectInspector);
+    StandardMapObjectInspector result = cachedStandardMapObjectInspector.get(signature);
+    if (result == null) {
+      result = new StandardMapObjectInspector(mapKeyObjectInspector, mapValueObjectInspector);
+      cachedStandardMapObjectInspector.put(signature, result);
+    }
+    return result;
+  }
+  
+  static HashMap<ArrayList<List<?>>, StandardStructObjectInspector> cachedStandardStructObjectInspector =
+    new HashMap<ArrayList<List<?>>, StandardStructObjectInspector>(); 
+  public static StandardStructObjectInspector getStandardStructObjectInspector(List<String> structFieldNames, List<ObjectInspector> structFieldObjectInspectors) {
+    ArrayList<List<?>> signature = new ArrayList<List<?>>();
+    signature.add(structFieldNames);
+    signature.add(structFieldObjectInspectors);
+    StandardStructObjectInspector result = cachedStandardStructObjectInspector.get(signature);
+    if (result == null) {
+      result = new StandardStructObjectInspector(structFieldNames, structFieldObjectInspectors);
+      cachedStandardStructObjectInspector.put(signature, result);
+    }
+    return result;
+  }
+  
+  static HashMap<List<StructObjectInspector>, UnionStructObjectInspector> cachedUnionStructObjectInspector =
+    new HashMap<List<StructObjectInspector>, UnionStructObjectInspector>(); 
+  public static UnionStructObjectInspector getUnionStructObjectInspector(List<StructObjectInspector> structObjectInspectors) {
+    UnionStructObjectInspector result = cachedUnionStructObjectInspector.get(structObjectInspectors);
+    if (result == null) {
+      result = new UnionStructObjectInspector(structObjectInspectors);
+      cachedUnionStructObjectInspector.put(structObjectInspectors, result);
+    }
+    return result;
+  }
+  
+  
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java

@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * ObjectInspectorFactory is the primary way to create new ObjectInspector instances.
+ * 
+ * SerDe classes should call the static functions in this library to create an ObjectInspector
+ * to return to the caller of SerDe2.getObjectInspector(). 
+ */
+public class ObjectInspectorUtils {
+
+  /** This function defines the list of PrimitiveClasses that we support. 
+   *  A PrimitiveClass should support java serialization/deserialization.
+   */
+  public static boolean isPrimitiveClass(Class<?> c) {
+    return ((c == String.class) || (c == Boolean.class) ||
+            (c == Character.class) || (c == java.sql.Date.class) || 
+            java.lang.Number.class.isAssignableFrom(c) ||
+            c.isPrimitive());
+  }
+  
+  /**
+   * Generalize the Java primitive types to the corresponding 
+   * Java Classes.  
+   */
+  public static Class<?> generalizePrimitive(Class<?> primitiveClass) {
+    if (primitiveClass == Boolean.TYPE)   primitiveClass = Boolean.class;
+    if (primitiveClass == Byte.TYPE)      primitiveClass = Byte.class;
+    if (primitiveClass == Character.TYPE) primitiveClass = Character.class;
+    if (primitiveClass == Short.TYPE)     primitiveClass = Short.class;
+    if (primitiveClass == Integer.TYPE)   primitiveClass = Integer.class;
+    if (primitiveClass == Long.TYPE)      primitiveClass = Long.class;
+    if (primitiveClass == Float.TYPE)     primitiveClass = Float.class;
+    if (primitiveClass == Double.TYPE)    primitiveClass = Double.class;
+    if (primitiveClass == Void.TYPE)      primitiveClass = Void.class;
+    return primitiveClass;
+  }
+  
+
+  static ArrayList<ArrayList<String>> integerArrayCache = new ArrayList<ArrayList<String>>();
+  /**
+   * Returns an array of Integer strings, starting from "0".
+   * This function caches the arrays to provide a better performance. 
+   */
+  public static ArrayList<String> getIntegerArray(int size) {
+    while (integerArrayCache.size() <= size) {
+      integerArrayCache.add(null);
+    }
+    ArrayList<String> result = integerArrayCache.get(size);
+    if (result == null) {
+      result = new ArrayList<String>();
+      for (int i=0; i<size; i++) {
+        result.add(Integer.valueOf(i).toString());
+      }
+      integerArrayCache.set(size, result);
+    }
+    return result;
+  }
+
+  static ArrayList<String> integerCSVCache = new ArrayList<String>(); 
+  public static String getIntegerCSV(int size) {
+    while (integerCSVCache.size() <= size) {
+      integerCSVCache.add(null);
+    }
+    String result = integerCSVCache.get(size);
+    if (result == null) {
+      StringBuilder sb = new StringBuilder();
+      for(int i=0; i<size; i++) {
+        if (i>0) sb.append(",");
+        sb.append("" + i);
+      }
+      result = sb.toString();
+      integerCSVCache.set(size, result);
+    }
+    return result;
+  }
+  
+
+  /**
+   * Get the standard ObjectInspector for an ObjectInspector.
+   * 
+   * The returned ObjectInspector can be used to inspect the standard object.
+   */
+  public static ObjectInspector getStandardObjectInspector(ObjectInspector oi) {
+    ObjectInspector result = null;
+    switch (oi.getCategory()) {
+      case PRIMITIVE: {
+        PrimitiveObjectInspector poi =(PrimitiveObjectInspector)oi;
+        result = poi;
+        break;
+      }
+      case LIST: {
+        ListObjectInspector loi = (ListObjectInspector)oi;
+        result = ObjectInspectorFactory.getStandardListObjectInspector(loi.getListElementObjectInspector());
+        break;
+      }
+      case MAP: {
+        MapObjectInspector moi = (MapObjectInspector)oi;
+        result = ObjectInspectorFactory.getStandardMapObjectInspector(
+            moi.getMapKeyObjectInspector(),
+            moi.getMapValueObjectInspector());
+        break;
+      }
+      case STRUCT: {
+        StructObjectInspector soi = (StructObjectInspector)oi;
+        List<? extends StructField> fields = soi.getAllStructFieldRefs();
+        List<String> fieldNames = new ArrayList<String>(fields.size());
+        List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fields.size());
+        for(StructField f : fields) {
+          fieldNames.add(f.getFieldName());
+          fieldObjectInspectors.add(f.getFieldObjectInspector());
+        }
+        result = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
+        break;
+      }
+      default: {
+        throw new RuntimeException("Unknown ObjectInspector category!");
+      }
+    }
+    return result;
+  }
+  
+  // TODO: should return o if the ObjectInspector is a standard ObjectInspector hierarchy
+  // (all internal ObjectInspector needs to be standard ObjectInspectors)
+  public static Object getStandardObject(Object o, ObjectInspector oi) {
+    if (o == null) {
+      return null;
+    }
+    
+    Object result = null;
+    switch (oi.getCategory()) {
+      case PRIMITIVE: {
+        result = o;
+        break;
+      }
+      case LIST: {
+        ListObjectInspector loi = (ListObjectInspector)oi;
+        int length = loi.getListLength(o);
+        ArrayList<Object> list = new ArrayList<Object>(length);
+        for(int i=0; i<length; i++) {
+          list.add(getStandardObject(
+              loi.getListElement(o, i),
+              loi.getListElementObjectInspector()));
+        }
+        result = list;
+        break;
+      }
+      case MAP: {
+        MapObjectInspector moi = (MapObjectInspector)oi;
+        HashMap<Object, Object> map = new HashMap<Object, Object>();
+        Map<? extends Object, ? extends Object> omap = moi.getMap(o);
+        for(Map.Entry<? extends Object, ? extends Object> entry: omap.entrySet()) {
+          map.put(getStandardObject(entry.getKey(), moi.getMapKeyObjectInspector()),
+              getStandardObject(entry.getValue(), moi.getMapValueObjectInspector()));
+        }
+        result = map;
+        break;
+      }
+      case STRUCT: {
+        StructObjectInspector soi = (StructObjectInspector)oi;
+        List<? extends StructField> fields = soi.getAllStructFieldRefs();
+        ArrayList<Object> struct = new ArrayList<Object>(fields.size()); 
+        for(StructField f : fields) {
+          struct.add(getStandardObject(soi.getStructFieldData(o, f), f.getFieldObjectInspector()));
+        }
+        result = struct;
+        break;
+      }
+      default: {
+        throw new RuntimeException("Unknown ObjectInspector category!");
+      }
+    }
+    return result;
+  }  
+  
+  public static String getStandardStructTypeName(StructObjectInspector soi) {
+    StringBuilder sb = new StringBuilder();
+    sb.append("struct{");
+    List<? extends StructField> fields = soi.getAllStructFieldRefs(); 
+    for(int i=0; i<fields.size(); i++) {
+      if (i>0) sb.append(",");
+      sb.append(fields.get(i).getFieldName());
+      sb.append(":");
+      sb.append(fields.get(i).getFieldObjectInspector().getTypeName());
+    }
+    sb.append("}");
+    return sb.toString();
+  }
+  
+  public static StructField getStandardStructFieldRef(String fieldName, List<? extends StructField> fields) {
+    fieldName = fieldName.toLowerCase();
+    for(int i=0; i<fields.size(); i++) {
+      if (fields.get(i).getFieldName().equals(fieldName)) {
+        return fields.get(i);
+      }
+    }
+    throw new RuntimeException("cannot find field " + fieldName + " from " + fields); 
+    // return null;
+  }
+
+  
+  
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java

@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+public interface PrimitiveObjectInspector extends ObjectInspector{
+
+  public Class<?> getPrimitiveClass();
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java

@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * ReflectionStructObjectInspector works on struct data that is stored as a native Java object.
+ * It will drill down into the Java class to get the fields and construct ObjectInspectors for 
+ * the fields, if they are not specified.
+ * 
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead
+ * of directly creating an instance of this class. 
+ *
+ */
+public class ReflectionStructObjectInspector implements StructObjectInspector {
+
+  public static class MyField implements StructField {
+    protected Field field;
+    protected ObjectInspector fieldObjectInspector;
+    
+    public MyField(Field field, ObjectInspector fieldObjectInspector) {
+      this.field = field;
+      this.fieldObjectInspector = fieldObjectInspector;
+    }
+    public String getFieldName() {
+      return field.getName().toLowerCase();
+    }
+    public ObjectInspector getFieldObjectInspector() {
+      return fieldObjectInspector;
+    }
+    public String toString() {
+      return field.toString();
+    }
+  }
+
+  Class<?> objectClass;
+  List<MyField> fields;
+  
+  public ReflectionStructObjectInspector(Class<?> objectClass, List<ObjectInspector> structFieldObjectInspectors) {
+    init(objectClass, structFieldObjectInspectors);
+  }
+
+  public Category getCategory() {
+    return Category.STRUCT;
+  }
+
+  public String getTypeName() {
+    return objectClass.getName();
+  }
+  
+  /**
+   * This method is only intended to be used by Utilities class in this package.
+   */
+  ReflectionStructObjectInspector() {
+  }
+  
+  /**
+   * This method is only intended to be used by Utilities class in this package.
+   * The reason that this method is not recursive by itself is because we want to allow
+   * recursive types.
+   */
+  void init(Class<?> objectClass, List<ObjectInspector> structFieldObjectInspectors) {
+    assert(!List.class.isAssignableFrom(objectClass));
+    assert(!Map.class.isAssignableFrom(objectClass));
+    
+    this.objectClass = objectClass;
+    Field[] reflectionFields = objectClass.getDeclaredFields();
+    fields = new ArrayList<MyField>(structFieldObjectInspectors.size());
+    int used = 0;
+    for (int i=0; i<reflectionFields.length; i++) {
+      if (!shouldIgnoreField(reflectionFields[i].getName())) {
+        fields.add(new MyField(reflectionFields[i], structFieldObjectInspectors.get(used++)));
+      }
+    }
+    assert(fields.size() == structFieldObjectInspectors.size());
+  }
+  
+  // ThriftStructObjectInspector will override and ignore __isset fields. 
+  public boolean shouldIgnoreField(String name) {
+    return false;
+  }
+  
+  // Without Data
+  public StructField getStructFieldRef(String fieldName) {
+    return ObjectInspectorUtils.getStandardStructFieldRef(fieldName, fields);
+  }
+  public List<? extends StructField> getAllStructFieldRefs() {
+    return fields;
+  }
+
+  // With Data
+  public Object getStructFieldData(Object data, StructField fieldRef) {
+    if (data == null) {
+      return null;
+    }
+    try {
+      MyField f = (MyField) fieldRef;
+      return f.field.get(data);
+    } catch (Exception e) {
+      throw new RuntimeException(e); 
+    }
+  }
+  public List<Object> getStructFieldsDataAsList(Object data) {
+    if (data == null) {
+      return null;
+    }
+    try {
+      ArrayList<Object> result = new ArrayList<Object>(fields.size());
+      for(int i=0; i<fields.size(); i++) {
+        result.add(fields.get(i).field.get(data));
+      }
+      return result;
+    } catch (Exception e) {
+      throw new RuntimeException(e); 
+    }
+  }
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardListObjectInspector.java

@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.List;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+/**
+ * DefaultListObjectInspector works on list data that is stored as a Java List or Java Array object.
+ * 
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead
+ * of directly creating an instance of this class. 
+ */
+class StandardListObjectInspector implements ListObjectInspector {
+
+  ObjectInspector listElementObjectInspector;
+  
+  /** Call ObjectInspectorFactory.getStandardListObjectInspector instead.
+   */
+  protected StandardListObjectInspector(ObjectInspector listElementObjectInspector) {
+    this.listElementObjectInspector = listElementObjectInspector;
+  }
+
+  public final Category getCategory() {
+    return Category.LIST;
+  }
+
+  // without data
+  public ObjectInspector getListElementObjectInspector() {
+    return listElementObjectInspector;
+  }
+  
+  // with data
+  public Object getListElement(Object data, int index) {
+    List<?> list = getList(data);
+    if (list == null || index < 0 || index >= list.size()) {
+      return null;
+    }
+    return list.get(index);
+  }
+  
+  public int getListLength(Object data) {
+    List<?> list = getList(data);
+    if (list == null) return -1;
+    return list.size();
+  }
+  
+  public List<?> getList(Object data) {
+    if (data == null) return null;
+    if (data.getClass().isArray()) {
+      data = java.util.Arrays.asList((Object[])data);
+    }
+    List<?> list = (List<?>) data;
+    return list;
+  }
+
+  public String getTypeName() {
+    return "list<" + listElementObjectInspector.getTypeName() + ">";
+  }
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardMapObjectInspector.java

@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.Map;
+
+/**
+ * StandardMapObjectInspector works on map data that is stored as a Java Map object.
+ * Note: the key object of the map must support equals and hashCode by itself.
+ * 
+ * We also plan to have a GeneralMapObjectInspector which can work on map with 
+ * key objects that does not support equals and hashCode.  That will require us to 
+ * store InspectableObject as the key, which will have overridden equals and hashCode 
+ * methods.
+ * 
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead
+ * of directly creating an instance of this class. 
+ */
+class StandardMapObjectInspector implements MapObjectInspector {
+
+  ObjectInspector mapKeyObjectInspector;
+  ObjectInspector mapValueObjectInspector;
+  
+  /** Call ObjectInspectorFactory.getStandardMapObjectInspector instead.
+   */
+  protected StandardMapObjectInspector(ObjectInspector mapKeyObjectInspector, ObjectInspector mapValueObjectInspector) {
+    this.mapKeyObjectInspector = mapKeyObjectInspector;
+    this.mapValueObjectInspector = mapValueObjectInspector;
+  }
+
+  // without data
+  public ObjectInspector getMapKeyObjectInspector() {
+    return mapKeyObjectInspector;
+  }
+  public ObjectInspector getMapValueObjectInspector() {
+    return mapValueObjectInspector;
+  }
+
+  // with data
+  // TODO: Now we assume the key Object supports hashCode and equals functions.
+  public Object getMapValueElement(Object data, Object key) {
+    if (data == null || key == null) return null;
+    Map<?,?> map = (Map<?,?>)data;
+    return map.get(key);
+  }
+  int getMapSize(Object data) {
+    if (data == null) return -1;
+    Map<?,?> map = (Map<?,?>)data;
+    return map.size();
+  }
+  public Map<?,?> getMap(Object data) {
+    if (data == null) return null;
+    Map<?,?> map = (Map<?,?>)data;
+    return map;
+  }
+
+  public final Category getCategory() {
+    return Category.MAP;
+  }
+
+  public String getTypeName() {
+    return "map<" + mapKeyObjectInspector.getTypeName() + "," 
+        + mapValueObjectInspector.getTypeName() + ">";
+  }
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardPrimitiveObjectInspector.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+/**
+ * PrimitiveObjectInspector works on primitive data that is stored as a Primitive Java object.
+ * 
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead
+ * of directly creating an instance of this class. 
+ */
+class StandardPrimitiveObjectInspector implements PrimitiveObjectInspector {
+
+  Class<?> primitiveClass;
+  
+  /** Call ObjectInspectorFactory.getStandardPrimitiveObjectInspector instead.
+   */
+  protected StandardPrimitiveObjectInspector(Class<?> primitiveClass) {
+    this.primitiveClass = primitiveClass;
+  }
+
+  public Class<?> getPrimitiveClass() {
+    return primitiveClass;
+  }
+
+  public final Category getCategory() {
+    return Category.PRIMITIVE;
+  }
+
+  public String getTypeName() {
+    return primitiveClass.getName();
+  }
+
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java

@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * ListStructObjectInspector works on struct data that is stored as a Java List or Java Array object.
+ * Basically, the fields are stored sequentially in the List object.
+ * 
+ * The names of the struct fields and the internal structure of the struct fields are specified in 
+ * the ctor of the StructObjectInspector.
+ * 
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead
+ * of directly creating an instance of this class. 
+ */
+public class StandardStructObjectInspector implements StructObjectInspector {
+
+  public static final Log LOG = LogFactory.getLog(StandardStructObjectInspector.class.getName());
+  
+  protected static class MyField implements StructField {
+    protected int fieldID;
+    protected String fieldName;
+    protected ObjectInspector fieldObjectInspector;
+    
+    public MyField(int fieldID, String fieldName, ObjectInspector fieldObjectInspector) {
+      this.fieldID = fieldID;
+      this.fieldName = fieldName.toLowerCase();
+      this.fieldObjectInspector = fieldObjectInspector;
+    }
+
+    public int getFieldID() {
+      return fieldID;
+    }
+    public String getFieldName() {
+      return fieldName;
+    }
+    public ObjectInspector getFieldObjectInspector() {
+      return fieldObjectInspector;
+    }
+    
+    public String toString() {
+      return "" + fieldID + ":" + fieldName;
+    }
+  }
+  
+  protected List<MyField> fields;
+  
+  public String getTypeName() {
+    return ObjectInspectorUtils.getStandardStructTypeName(this);
+  }
+  
+  /** Call ObjectInspectorFactory.getStandardListObjectInspector instead.
+   */
+  protected StandardStructObjectInspector(List<String> structFieldNames, List<ObjectInspector> structFieldObjectInspectors) {
+    init(structFieldNames, structFieldObjectInspectors);
+  }
+  protected void init(List<String> structFieldNames, List<ObjectInspector> structFieldObjectInspectors) {
+    assert(structFieldNames.size() == structFieldObjectInspectors.size());
+    
+    fields = new ArrayList<MyField>(structFieldNames.size()); 
+    for(int i=0; i<structFieldNames.size(); i++) {
+      fields.add(new MyField(i, structFieldNames.get(i), structFieldObjectInspectors.get(i)));
+    }
+  }
+  
+  protected StandardStructObjectInspector(List<StructField> fields) {
+    init(fields);
+  }
+  protected void init(List<StructField> fields) {
+    this.fields = new ArrayList<MyField>(fields.size()); 
+    for(int i=0; i<fields.size(); i++) {
+      this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields.get(i).getFieldObjectInspector()));
+    }
+  }
+
+  
+  public final Category getCategory() {
+    return Category.STRUCT;
+  }
+
+  // Without Data
+  public StructField getStructFieldRef(String fieldName) {
+    return ObjectInspectorUtils.getStandardStructFieldRef(fieldName, fields);
+  }
+  public List<? extends StructField> getAllStructFieldRefs() {
+    return fields;
+  }
+
+  boolean warned = false;
+  // With Data
+  @SuppressWarnings("unchecked")
+  public Object getStructFieldData(Object data, StructField fieldRef) {
+    List<Object> list = getStructFieldsDataAsList(data);
+    if (list == null) {
+      return null;
+    }
+    MyField f = (MyField) fieldRef;
+    if (fields.size() != list.size() && !warned) {
+      // TODO: remove this
+      warned = true;
+      LOG.warn("Trying to access " + fields.size() + " fields inside a list of " + list.size()
+          + " elements: " + list);
+      LOG.warn("ignoring similar errors.");
+    }
+    int fieldID = f.getFieldID();
+    assert(fieldID >= 0 && fieldID < fields.size());
+    
+    return fieldID >= list.size() ? null : list.get(fieldID);
+  }
+  @SuppressWarnings("unchecked")
+  public List<Object> getStructFieldsDataAsList(Object data) {
+    if (data == null) {
+      return null;
+    }
+    if (data.getClass().isArray()) {
+      data = java.util.Arrays.asList((Object[])data);
+    }
+    List<Object> list = (List<Object>) data;
+    assert(list.size() == fields.size());
+    return list;
+  }
+
+  
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java

@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+/**
+ * StructField is an empty interface.
+ * 
+ * Classes implementing this interface are considered to represent 
+ * a field of a struct for this serde package.
+ */
+public interface StructField {
+
+  /**
+   * Get the name of the field.  The name should be always in lower-case.
+   */
+  String getFieldName();
+  
+  /**
+   * Get the ObjectInspector for the field.
+   */
+  ObjectInspector getFieldObjectInspector();
+  
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java

@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.List;
+
+
+public interface StructObjectInspector extends ObjectInspector {
+
+  // ** Methods that does not need a data object **
+  /** Returns all the fields. 
+   */
+  public List<? extends StructField> getAllStructFieldRefs();
+
+  /** Look up a field.
+   */
+  public StructField getStructFieldRef(String fieldName);
+
+  // ** Methods that need a data object **
+  /** returns null for data = null.
+   */
+  public Object getStructFieldData(Object data, StructField fieldRef);
+
+  /** returns null for data = null.
+   */
+  public List<Object> getStructFieldsDataAsList(Object data);
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftStructObjectInspector.java

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+/**
+ * 
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead
+ * of directly creating an instance of this class. 
+ */
+class ThriftStructObjectInspector extends
+    ReflectionStructObjectInspector {
+
+  public boolean shouldIgnoreField(String name) {
+    return "__isset".equals(name);
+  }
+  
+  public boolean equals(Object b) {
+    if (this == b) return true;
+    if (!b.getClass().equals(this.getClass())) {
+      return false;
+    }
+    ThriftStructObjectInspector bInspector = (ThriftStructObjectInspector)b;
+    return objectClass.equals(bInspector.objectClass);
+  }
+
+  public int hashCode() {
+    return 7 * objectClass.hashCode(); 
+  }
+    
+}

src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java

@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * UnionStructObjectInspector unions several struct data into a single struct.
+ * Basically, the fields of these structs are put together sequentially into a single struct.
+ * 
+ * The object that can be acceptable by this ObjectInspector is a List of objects, each of
+ * which can be inspected by the ObjectInspector provided in the ctor of UnionStructObjectInspector.
+ * 
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead
+ * of directly creating an instance of this class. 
+ */
+public class UnionStructObjectInspector implements StructObjectInspector {
+
+  public static class MyField implements StructField {
+    public int structID;
+    StructField structField;
+    public MyField(int structID, StructField structField) {
+      this.structID = structID;
+      this.structField = structField;
+    }
+    public String getFieldName() {
+      return structField.getFieldName();
+    }
+    public ObjectInspector getFieldObjectInspector() {
+      return structField.getFieldObjectInspector();
+    }
+  }
+  
+  List<StructObjectInspector> unionObjectInspectors;
+  List<MyField> fields;
+  
+  protected UnionStructObjectInspector(List<StructObjectInspector> unionObjectInspectors) {
+    init(unionObjectInspectors);
+  }
+
+  void init(List<StructObjectInspector> unionObjectInspectors) {
+    this.unionObjectInspectors = unionObjectInspectors;
+    
+    int totalSize = 0;
+    for (int i=0; i<unionObjectInspectors.size(); i++) {
+      totalSize += unionObjectInspectors.get(i).getAllStructFieldRefs().size();
+    }
+    
+    fields = new ArrayList<MyField>(totalSize); 
+    for (int i=0; i<unionObjectInspectors.size(); i++) {
+      StructObjectInspector oi = unionObjectInspectors.get(i);
+      for(StructField sf: oi.getAllStructFieldRefs()) {
+        fields.add(new MyField(i, sf));
+      }
+    }
+  }
+  
+  
+  public final Category getCategory() {
+    return Category.STRUCT;
+  }
+
+  public String getTypeName() {
+    return ObjectInspectorUtils.getStandardStructTypeName(this);
+  }
+
+  // Without Data
+  public StructField getStructFieldRef(String fieldName) {
+    return ObjectInspectorUtils.getStandardStructFieldRef(fieldName, fields);
+  }
+  public List<? extends StructField> getAllStructFieldRefs() {
+    return fields;
+  }
+
+  // With Data
+  @SuppressWarnings("unchecked")
+  public Object getStructFieldData(Object data, StructField fieldRef) {
+    if (data == null) {
+      return null;
+    }
+    if (data.getClass().isArray()) {
+      data = java.util.Arrays.asList((Object[])data);
+    }
+    MyField f = (MyField) fieldRef;
+    List<Object> list = (List<Object>) data;
+    assert(list.size() == unionObjectInspectors.size());
+    return unionObjectInspectors.get(f.structID).getStructFieldData(list.get(f.structID), f.structField);
+  }
+  @SuppressWarnings("unchecked")
+  public List<Object> getStructFieldsDataAsList(Object data) {
+    if (data == null) {
+      return null;
+    }
+    if (data.getClass().isArray()) {
+      data = java.util.Arrays.asList((Object[])data);
+    }
+    List<Object> list = (List<Object>) data;
+    assert(list.size() == unionObjectInspectors.size());
+    // Explode
+    ArrayList<Object> result = new ArrayList<Object>(fields.size());
+    for(int i=0; i<unionObjectInspectors.size(); i++) {
+      result.addAll(unionObjectInspectors.get(i).getStructFieldsDataAsList(list.get(i)));
+    }
+    return result;
+  }
+
+}

src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/MyStruct.java

@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * Class for testing TestReflectionObjectInspector.
+ */
+public class MyStruct {
+    public int myInt;
+    public Integer myInteger;
+    public String myString;
+    // Note: this is a recursive struct
+    public MyStruct myStruct;
+    public List<String> myListString;
+    public Map<String, String> myMapStringString;
+}

src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java

@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.thrift_test.Complex;
+import org.apache.hadoop.hive.serde2.thrift_test.IntString;
+
+import junit.framework.TestCase;
+
+public class TestObjectInspectorUtils extends TestCase {
+
+  public void testObjectInspectorUtils() throws Throwable {
+    try {
+      ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(
+          Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
+      
+      // metadata
+      assertEquals(Category.STRUCT, oi1.getCategory());
+      // standard ObjectInspector
+      StructObjectInspector soi = (StructObjectInspector)ObjectInspectorUtils.getStandardObjectInspector(oi1);
+      List<? extends StructField> fields = soi.getAllStructFieldRefs();
+      assertEquals(6, fields.size());
+      assertEquals(fields.get(0), soi.getStructFieldRef("aint"));
+  
+      // null
+      for (int i=0; i<fields.size(); i++) {
+        assertNull(soi.getStructFieldData(null, fields.get(i)));
+      }
+      
+      // real object
+      Complex cc = new Complex();
+      cc.aint = 1;
+      cc.astring = "test";
+      List<Integer> c2 = Arrays.asList(new Integer[]{1,2,3}); 
+      cc.lint =  c2;
+      List<String> c3 = Arrays.asList(new String[]{"one", "two"});
+      cc.lstring =  c3;
+      List<IntString> c4 = new ArrayList<IntString>(); 
+      cc.lintstring = c4;
+      cc.mstringstring = null; 
+      // standard object
+      Object c = ObjectInspectorUtils.getStandardObject(cc, oi1);
+      
+      assertEquals(1, soi.getStructFieldData(c, fields.get(0)));
+      assertEquals("test", soi.getStructFieldData(c, fields.get(1)));
+      assertEquals(c2, soi.getStructFieldData(c, fields.get(2)));
+      assertEquals(c3, soi.getStructFieldData(c, fields.get(3)));
+      assertEquals(c4, soi.getStructFieldData(c, fields.get(4)));
+      assertNull(soi.getStructFieldData(c, fields.get(5)));
+      ArrayList<Object> cfields = new ArrayList<Object>();
+      for(int i=0; i<6; i++) {
+        cfields.add(soi.getStructFieldData(c, fields.get(i)));
+      }
+      assertEquals(cfields, soi.getStructFieldsDataAsList(c));
+      
+      // sub fields
+      assertEquals(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class),
+          fields.get(0).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class),
+          fields.get(1).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class)),
+          fields.get(2).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class)),
+          fields.get(3).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(
+          ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, 
+              ObjectInspectorFactory.ObjectInspectorOptions.THRIFT)),
+          fields.get(4).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector(
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class),
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class)),
+          fields.get(5).getFieldObjectInspector());
+    } catch (Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+    
+  }  
+}

src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java

@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+
+import junit.framework.TestCase;
+
+
+public class TestReflectionObjectInspectors extends TestCase {
+
+  public void testReflectionObjectInspectors() throws Throwable {
+    try {
+      ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(
+          MyStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+      ObjectInspector oi2 = ObjectInspectorFactory.getReflectionObjectInspector(
+          MyStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+      assertEquals(oi1, oi2);
+      
+      // metadata
+      assertEquals(Category.STRUCT, oi1.getCategory());
+      StructObjectInspector soi = (StructObjectInspector)oi1;
+      List<? extends StructField> fields = soi.getAllStructFieldRefs(); 
+      assertEquals(6, fields.size());
+      assertEquals(fields.get(2), soi.getStructFieldRef("myString"));
+  
+      // null
+      for (int i=0; i<fields.size(); i++) {
+        assertNull(soi.getStructFieldData(null, fields.get(i)));
+      }
+      assertNull(soi.getStructFieldsDataAsList(null));
+      
+      // non nulls
+      MyStruct a = new MyStruct();
+      a.myInt = 1;
+      a.myInteger = 2;
+      a.myString = "test";
+      a.myStruct = a;
+      a.myListString = Arrays.asList(new String[]{"a", "b", "c"});
+      a.myMapStringString = new HashMap<String, String>();
+      a.myMapStringString.put("key", "value");
+      
+      assertEquals(1, soi.getStructFieldData(a, fields.get(0)));
+      assertEquals(2, soi.getStructFieldData(a, fields.get(1)));
+      assertEquals("test", soi.getStructFieldData(a, fields.get(2)));
+      assertEquals(a, soi.getStructFieldData(a, fields.get(3)));
+      assertEquals(a.myListString, soi.getStructFieldData(a, fields.get(4)));
+      assertEquals(a.myMapStringString, soi.getStructFieldData(a, fields.get(5)));
+      ArrayList<Object> afields = new ArrayList<Object>();
+      for(int i=0; i<6; i++) {
+        afields.add(soi.getStructFieldData(a, fields.get(i)));
+      }
+      assertEquals(afields, soi.getStructFieldsDataAsList(a));
+      
+      // sub fields
+      assertEquals(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class),
+          fields.get(0).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class),
+          fields.get(1).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class),
+          fields.get(2).getFieldObjectInspector());
+      assertEquals(soi, fields.get(3).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class)),
+          fields.get(4).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector(
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class),
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class)),
+          fields.get(5).getFieldObjectInspector());
+    } catch (Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+}

src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java

@@ -0,0 +1,207 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+
+import junit.framework.TestCase;
+
+public class TestStandardObjectInspectors extends TestCase {
+
+  
+  void doTestStandardPrimitiveObjectInspector(Class<?> c) throws Throwable {
+    try {
+      StandardPrimitiveObjectInspector oi1 = ObjectInspectorFactory.getStandardPrimitiveObjectInspector(c);
+      StandardPrimitiveObjectInspector oi2 = ObjectInspectorFactory.getStandardPrimitiveObjectInspector(c);
+      assertEquals(oi1, oi2);
+      assertEquals(Category.PRIMITIVE, oi1.getCategory());
+      assertEquals(c, oi1.getPrimitiveClass());
+      assertEquals(c.getName(), oi1.getTypeName()); 
+    } catch (Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  public void testStandardPrimitiveObjectInspector() throws Throwable {
+    try {
+      doTestStandardPrimitiveObjectInspector(Boolean.class);
+      doTestStandardPrimitiveObjectInspector(Byte.class);
+      doTestStandardPrimitiveObjectInspector(Integer.class);
+      doTestStandardPrimitiveObjectInspector(Long.class);
+      doTestStandardPrimitiveObjectInspector(Float.class);
+      doTestStandardPrimitiveObjectInspector(Double.class);
+      doTestStandardPrimitiveObjectInspector(String.class);
+      doTestStandardPrimitiveObjectInspector(java.sql.Date.class);
+    } catch (Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  public void testStandardListObjectInspector() throws Throwable {
+    try {
+      StandardListObjectInspector loi1 = ObjectInspectorFactory.getStandardListObjectInspector(
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class));
+      StandardListObjectInspector loi2 = ObjectInspectorFactory.getStandardListObjectInspector(
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class));
+      assertEquals(loi1, loi2);
+      
+      // metadata
+      assertEquals(Category.LIST, loi1.getCategory());
+      assertEquals(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class),
+          loi1.getListElementObjectInspector());
+      
+      // null
+      assertNull("loi1.getList(null) should be null.", loi1.getList(null));
+      assertEquals("loi1.getListLength(null) should be -1.", loi1.getListLength(null), -1);
+      assertNull("loi1.getListElement(null, 0) should be null", loi1.getListElement(null, 0));
+      assertNull("loi1.getListElement(null, 100) should be null", loi1.getListElement(null, 100));
+      
+      // ArrayList
+      ArrayList<Integer> list = new ArrayList<Integer>();
+      list.add(0);
+      list.add(1);
+      list.add(2);
+      list.add(3);
+      assertEquals(4, loi1.getList(list).size());
+      assertEquals(4, loi1.getListLength(list));
+      assertEquals(0, loi1.getListElement(list, 0));
+      assertEquals(3, loi1.getListElement(list, 3));
+      assertNull(loi1.getListElement(list, -1));
+      assertNull(loi1.getListElement(list, 4));
+    } catch (Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+      
+  }
+
+  public void testStandardMapObjectInspector() throws Throwable {
+    try {
+      StandardMapObjectInspector moi1 = ObjectInspectorFactory.getStandardMapObjectInspector(
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class),
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class)
+          );
+      StandardMapObjectInspector moi2 = ObjectInspectorFactory.getStandardMapObjectInspector(
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class),
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class)
+          );
+      assertEquals(moi1, moi2);
+      
+      // metadata
+      assertEquals(Category.MAP, moi1.getCategory());
+      assertEquals(moi1.getMapKeyObjectInspector(), 
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class));
+      assertEquals(moi2.getMapValueObjectInspector(), 
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class));
+      
+      // null
+      assertNull(moi1.getMap(null));
+      assertNull(moi1.getMapValueElement(null, null));
+      assertNull(moi1.getMapValueElement(null, "nokey"));
+      assertEquals(-1, moi1.getMapSize(null));
+      assertEquals("map<" 
+          + ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class).getTypeName() + ","
+          + ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class).getTypeName() + ">",
+          moi1.getTypeName());
+  
+      // HashMap
+      HashMap<String, Integer> map = new HashMap<String, Integer>();
+      map.put("one", 1);
+      map.put("two", 2);
+      map.put("three", 3);
+      assertEquals(map, moi1.getMap(map));
+      assertEquals(3, moi1.getMapSize(map));
+      assertEquals(1, moi1.getMapValueElement(map, "one"));
+      assertEquals(2, moi1.getMapValueElement(map, "two"));
+      assertEquals(3, moi1.getMapValueElement(map, "three"));
+      assertNull(moi1.getMapValueElement(map, null));
+      assertNull(moi1.getMapValueElement(map, "null"));
+    } catch (Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+      
+  }
+
+  
+  @SuppressWarnings("unchecked")
+  public void testStandardStructObjectInspector() throws Throwable {
+    try {
+      ArrayList<String> fieldNames = new ArrayList<String>();
+      fieldNames.add("firstInteger");
+      fieldNames.add("secondString");
+      fieldNames.add("thirdBoolean");
+      ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>();
+      fieldObjectInspectors.add(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class));
+      fieldObjectInspectors.add(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class));
+      fieldObjectInspectors.add(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Boolean.class));
+      
+      StandardStructObjectInspector soi1 = ObjectInspectorFactory.getStandardStructObjectInspector(
+          fieldNames, fieldObjectInspectors);
+      StandardStructObjectInspector soi2 = ObjectInspectorFactory.getStandardStructObjectInspector(
+          (ArrayList<String>)fieldNames.clone(), (ArrayList<ObjectInspector>)fieldObjectInspectors.clone());
+      assertEquals(soi1, soi2);
+      
+      // metadata
+      assertEquals(Category.STRUCT, soi1.getCategory());
+      List<? extends StructField> fields = soi1.getAllStructFieldRefs();
+      assertEquals(3, fields.size());
+      for (int i=0; i<3; i++) {
+        assertEquals(fieldNames.get(i).toLowerCase(), fields.get(i).getFieldName());
+        assertEquals(fieldObjectInspectors.get(i), fields.get(i).getFieldObjectInspector());
+      }
+      assertEquals(fields.get(1), soi1.getStructFieldRef("secondString"));
+      StringBuilder structTypeName = new StringBuilder(); 
+      structTypeName.append("struct{");
+      for(int i=0; i<fields.size(); i++) {
+        if (i>0) structTypeName.append(",");
+        structTypeName.append(fields.get(i).getFieldName());
+        structTypeName.append(":");
+        structTypeName.append(fields.get(i).getFieldObjectInspector().getTypeName());
+      }
+      structTypeName.append("}");
+      assertEquals(structTypeName.toString(), soi1.getTypeName());
+  
+      // null
+      assertNull(soi1.getStructFieldData(null, fields.get(0)));
+      assertNull(soi1.getStructFieldData(null, fields.get(1)));
+      assertNull(soi1.getStructFieldData(null, fields.get(2)));
+      assertNull(soi1.getStructFieldsDataAsList(null));
+  
+      // HashStruct
+      ArrayList<Object> struct = new ArrayList<Object>(3);
+      struct.add(1);
+      struct.add("two");
+      struct.add(true);
+      
+      assertEquals(1, soi1.getStructFieldData(struct, fields.get(0)));
+      assertEquals("two", soi1.getStructFieldData(struct, fields.get(1)));
+      assertEquals(true, soi1.getStructFieldData(struct, fields.get(2)));
+    } catch (Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+      
+  }
+  
+}

src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java

@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.hive.serde2.thrift_test.Complex;
+import org.apache.hadoop.hive.serde2.thrift_test.IntString;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+
+public class TestThriftObjectInspectors extends TestCase {
+
+  public void testThriftObjectInspectors() throws Throwable {
+
+    try {
+      ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(
+          Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
+      ObjectInspector oi2 = ObjectInspectorFactory.getReflectionObjectInspector(
+          Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
+      assertEquals(oi1, oi2);
+      
+      // metadata
+      assertEquals(Category.STRUCT, oi1.getCategory());
+      StructObjectInspector soi = (StructObjectInspector)oi1; 
+      List<? extends StructField> fields = soi.getAllStructFieldRefs();
+      assertEquals(6, fields.size());
+      assertEquals(fields.get(0), soi.getStructFieldRef("aint"));
+  
+      // null
+      for (int i=0; i<fields.size(); i++) {
+        assertNull(soi.getStructFieldData(null, fields.get(i)));
+      }
+      
+      // real object
+      Complex c = new Complex();
+      c.aint = 1;
+      c.astring = "test";
+      List<Integer> c2 = Arrays.asList(new Integer[]{1,2,3}); 
+      c.lint =  c2;
+      List<String> c3 = Arrays.asList(new String[]{"one", "two"});
+      c.lstring =  c3;
+      List<IntString> c4 = new ArrayList<IntString>(); 
+      c.lintstring = c4;
+      c.mstringstring = null; 
+      
+      assertEquals(1, soi.getStructFieldData(c, fields.get(0)));
+      assertEquals("test", soi.getStructFieldData(c, fields.get(1)));
+      assertEquals(c2, soi.getStructFieldData(c, fields.get(2)));
+      assertEquals(c3, soi.getStructFieldData(c, fields.get(3)));
+      assertEquals(c4, soi.getStructFieldData(c, fields.get(4)));
+      assertNull(soi.getStructFieldData(c, fields.get(5)));
+      ArrayList<Object> cfields = new ArrayList<Object>();
+      for(int i=0; i<6; i++) {
+        cfields.add(soi.getStructFieldData(c, fields.get(i)));
+      }
+      assertEquals(cfields, soi.getStructFieldsDataAsList(c));
+      
+      // sub fields
+      assertEquals(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class),
+          fields.get(0).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class),
+          fields.get(1).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class)),
+          fields.get(2).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class)),
+          fields.get(3).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(
+          ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, 
+              ObjectInspectorFactory.ObjectInspectorOptions.THRIFT)),
+          fields.get(4).getFieldObjectInspector());
+      assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector(
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class),
+          ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class)),
+          fields.get(5).getFieldObjectInspector());
+    } catch (Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+}

src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestUnionStructObjectInspector.java

@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+
+import junit.framework.TestCase;
+
+public class TestUnionStructObjectInspector extends TestCase {
+
+  
+  public void testUnionStructObjectInspector() throws Throwable {
+    try {
+      ArrayList<String> fieldNames1 = new ArrayList<String>();
+      fieldNames1.add("firstInteger");
+      fieldNames1.add("secondString");
+      fieldNames1.add("thirdBoolean");
+      ArrayList<ObjectInspector> fieldObjectInspectors1 = new ArrayList<ObjectInspector>();
+      fieldObjectInspectors1.add(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Integer.class));
+      fieldObjectInspectors1.add(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class));
+      fieldObjectInspectors1.add(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Boolean.class));
+      StandardStructObjectInspector soi1 = ObjectInspectorFactory.getStandardStructObjectInspector(
+          fieldNames1, fieldObjectInspectors1);
+  
+      ArrayList<String> fieldNames2 = new ArrayList<String>();
+      fieldNames2.add("fourthDate");
+      fieldNames2.add("fifthLong");
+      ArrayList<ObjectInspector> fieldObjectInspectors2 = new ArrayList<ObjectInspector>();
+      fieldObjectInspectors2.add(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(java.sql.Date.class));
+      fieldObjectInspectors2.add(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Long.class));
+      StandardStructObjectInspector soi2 = ObjectInspectorFactory.getStandardStructObjectInspector(
+          fieldNames2, fieldObjectInspectors2);
+      
+      UnionStructObjectInspector usoi1 = ObjectInspectorFactory.getUnionStructObjectInspector(
+          Arrays.asList(new StructObjectInspector[]{soi1, soi2}));
+      UnionStructObjectInspector usoi2 = ObjectInspectorFactory.getUnionStructObjectInspector(
+          Arrays.asList(new StructObjectInspector[]{soi1, soi2}));
+  
+      assertEquals(usoi1, usoi2);
+      
+      // metadata
+      assertEquals(Category.STRUCT, usoi1.getCategory());
+      List<? extends StructField> fields = usoi1.getAllStructFieldRefs();
+      assertEquals(5, fields.size());
+      for (int i=0; i<5; i++) {
+        if (i<=2) {
+          assertEquals(fieldNames1.get(i).toLowerCase(), fields.get(i).getFieldName());
+          assertEquals(fieldObjectInspectors1.get(i), fields.get(i).getFieldObjectInspector());
+        } else {
+          assertEquals(fieldNames2.get(i-3).toLowerCase(), fields.get(i).getFieldName());
+          assertEquals(fieldObjectInspectors2.get(i-3), fields.get(i).getFieldObjectInspector());
+        }
+      }
+      assertEquals(fields.get(1), usoi1.getStructFieldRef("secondString"));
+      assertEquals(fields.get(4), usoi1.getStructFieldRef("fifthLong"));
+      
+      // null
+      for (int i=0; i<5; i++) {
+        assertNull(usoi1.getStructFieldData(null, fields.get(i)));
+      }
+      
+      // real struct
+      ArrayList<Object> struct1 = new ArrayList<Object>(3);
+      struct1.add(1);
+      struct1.add("two");
+      struct1.add(true);
+      ArrayList<Object> struct2 = new ArrayList<Object>(2);
+      struct2.add(java.sql.Date.valueOf("2008-08-08"));
+      struct2.add(new Long(111));
+      ArrayList<Object> struct = new ArrayList<Object>(2);
+      struct.add(struct1);
+      struct.add(struct2);
+      ArrayList<Object> all = new ArrayList<Object>(5);
+      all.addAll(struct1);
+      all.addAll(struct2);
+      
+      for (int i=0; i<5; i++) {
+        assertEquals(all.get(i), usoi1.getStructFieldData(struct, fields.get(i)));
+      }
+    } catch (Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+}

src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/Complex.java

@@ -0,0 +1,363 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ */
+package org.apache.hadoop.hive.serde2.thrift_test;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import com.facebook.thrift.*;
+
+import com.facebook.thrift.protocol.*;
+import com.facebook.thrift.transport.*;
+
+public class Complex implements TBase, java.io.Serializable {
+  public int aint;
+  public String astring;
+  public List<Integer> lint;
+  public List<String> lstring;
+  public List<IntString> lintstring;
+  public Map<String,String> mstringstring;
+
+  public final Isset __isset = new Isset();
+  public static final class Isset implements java.io.Serializable {
+    public boolean aint = false;
+    public boolean astring = false;
+    public boolean lint = false;
+    public boolean lstring = false;
+    public boolean lintstring = false;
+    public boolean mstringstring = false;
+  }
+
+  public Complex() {
+  }
+
+  public Complex(
+    int aint,
+    String astring,
+    List<Integer> lint,
+    List<String> lstring,
+    List<IntString> lintstring,
+    Map<String,String> mstringstring)
+  {
+    this();
+    this.aint = aint;
+    this.__isset.aint = true;
+    this.astring = astring;
+    this.__isset.astring = true;
+    this.lint = lint;
+    this.__isset.lint = true;
+    this.lstring = lstring;
+    this.__isset.lstring = true;
+    this.lintstring = lintstring;
+    this.__isset.lintstring = true;
+    this.mstringstring = mstringstring;
+    this.__isset.mstringstring = true;
+  }
+
+  public boolean equals(Object that) {
+    if (that == null)
+      return false;
+    if (that instanceof Complex)
+      return this.equals((Complex)that);
+    return false;
+  }
+
+  public boolean equals(Complex that) {
+    if (that == null)
+      return false;
+
+    boolean this_present_aint = true;
+    boolean that_present_aint = true;
+    if (this_present_aint || that_present_aint) {
+      if (!(this_present_aint && that_present_aint))
+        return false;
+      if (this.aint != that.aint)
+        return false;
+    }
+
+    boolean this_present_astring = true && (this.astring != null);
+    boolean that_present_astring = true && (that.astring != null);
+    if (this_present_astring || that_present_astring) {
+      if (!(this_present_astring && that_present_astring))
+        return false;
+      if (!this.astring.equals(that.astring))
+        return false;
+    }
+
+    boolean this_present_lint = true && (this.lint != null);
+    boolean that_present_lint = true && (that.lint != null);
+    if (this_present_lint || that_present_lint) {
+      if (!(this_present_lint && that_present_lint))
+        return false;
+      if (!this.lint.equals(that.lint))
+        return false;
+    }
+
+    boolean this_present_lstring = true && (this.lstring != null);
+    boolean that_present_lstring = true && (that.lstring != null);
+    if (this_present_lstring || that_present_lstring) {
+      if (!(this_present_lstring && that_present_lstring))
+        return false;
+      if (!this.lstring.equals(that.lstring))
+        return false;
+    }
+
+    boolean this_present_lintstring = true && (this.lintstring != null);
+    boolean that_present_lintstring = true && (that.lintstring != null);
+    if (this_present_lintstring || that_present_lintstring) {
+      if (!(this_present_lintstring && that_present_lintstring))
+        return false;
+      if (!this.lintstring.equals(that.lintstring))
+        return false;
+    }
+
+    boolean this_present_mstringstring = true && (this.mstringstring != null);
+    boolean that_present_mstringstring = true && (that.mstringstring != null);
+    if (this_present_mstringstring || that_present_mstringstring) {
+      if (!(this_present_mstringstring && that_present_mstringstring))
+        return false;
+      if (!this.mstringstring.equals(that.mstringstring))
+        return false;
+    }
+
+    return true;
+  }
+
+  public int hashCode() {
+    return 0;
+  }
+
+  public void read(TProtocol iprot) throws TException {
+    TField field;
+    iprot.readStructBegin();
+    while (true)
+    {
+      field = iprot.readFieldBegin();
+      if (field.type == TType.STOP) { 
+        break;
+      }
+      switch (field.id)
+      {
+        case 1:
+          if (field.type == TType.I32) {
+            this.aint = iprot.readI32();
+            this.__isset.aint = true;
+          } else { 
+            TProtocolUtil.skip(iprot, field.type);
+          }
+          break;
+        case 2:
+          if (field.type == TType.STRING) {
+            this.astring = iprot.readString();
+            this.__isset.astring = true;
+          } else { 
+            TProtocolUtil.skip(iprot, field.type);
+          }
+          break;
+        case 3:
+          if (field.type == TType.LIST) {
+            {
+              TList _list0 = iprot.readListBegin();
+              this.lint = new ArrayList<Integer>(_list0.size);
+              for (int _i1 = 0; _i1 < _list0.size; ++_i1)
+              {
+                int _elem2 = 0;
+                _elem2 = iprot.readI32();
+                this.lint.add(_elem2);
+              }
+              iprot.readListEnd();
+            }
+            this.__isset.lint = true;
+          } else { 
+            TProtocolUtil.skip(iprot, field.type);
+          }
+          break;
+        case 4:
+          if (field.type == TType.LIST) {
+            {
+              TList _list3 = iprot.readListBegin();
+              this.lstring = new ArrayList<String>(_list3.size);
+              for (int _i4 = 0; _i4 < _list3.size; ++_i4)
+              {
+                String _elem5 = null;
+                _elem5 = iprot.readString();
+                this.lstring.add(_elem5);
+              }
+              iprot.readListEnd();
+            }
+            this.__isset.lstring = true;
+          } else { 
+            TProtocolUtil.skip(iprot, field.type);
+          }
+          break;
+        case 5:
+          if (field.type == TType.LIST) {
+            {
+              TList _list6 = iprot.readListBegin();
+              this.lintstring = new ArrayList<IntString>(_list6.size);
+              for (int _i7 = 0; _i7 < _list6.size; ++_i7)
+              {
+                IntString _elem8 = new IntString();
+                _elem8 = new IntString();
+                _elem8.read(iprot);
+                this.lintstring.add(_elem8);
+              }
+              iprot.readListEnd();
+            }
+            this.__isset.lintstring = true;
+          } else { 
+            TProtocolUtil.skip(iprot, field.type);
+          }
+          break;
+        case 6:
+          if (field.type == TType.MAP) {
+            {
+              TMap _map9 = iprot.readMapBegin();
+              this.mstringstring = new HashMap<String,String>(2*_map9.size);
+              for (int _i10 = 0; _i10 < _map9.size; ++_i10)
+              {
+                String _key11;
+                String _val12;
+                _key11 = iprot.readString();
+                _val12 = iprot.readString();
+                this.mstringstring.put(_key11, _val12);
+              }
+              iprot.readMapEnd();
+            }
+            this.__isset.mstringstring = true;
+          } else { 
+            TProtocolUtil.skip(iprot, field.type);
+          }
+          break;
+        default:
+          TProtocolUtil.skip(iprot, field.type);
+          break;
+      }
+      iprot.readFieldEnd();
+    }
+    iprot.readStructEnd();
+  }
+
+  public void write(TProtocol oprot) throws TException {
+    TStruct struct = new TStruct("Complex");
+    oprot.writeStructBegin(struct);
+    TField field = new TField();
+    field.name = "aint";
+    field.type = TType.I32;
+    field.id = 1;
+    oprot.writeFieldBegin(field);
+    oprot.writeI32(this.aint);
+    oprot.writeFieldEnd();
+    if (this.astring != null) {
+      field.name = "astring";
+      field.type = TType.STRING;
+      field.id = 2;
+      oprot.writeFieldBegin(field);
+      oprot.writeString(this.astring);
+      oprot.writeFieldEnd();
+    }
+    if (this.lint != null) {
+      field.name = "lint";
+      field.type = TType.LIST;
+      field.id = 3;
+      oprot.writeFieldBegin(field);
+      {
+        oprot.writeListBegin(new TList(TType.I32, this.lint.size()));
+        for (int _iter13 : this.lint)        {
+          oprot.writeI32(_iter13);
+        }
+        oprot.writeListEnd();
+      }
+      oprot.writeFieldEnd();
+    }
+    if (this.lstring != null) {
+      field.name = "lstring";
+      field.type = TType.LIST;
+      field.id = 4;
+      oprot.writeFieldBegin(field);
+      {
+        oprot.writeListBegin(new TList(TType.STRING, this.lstring.size()));
+        for (String _iter14 : this.lstring)        {
+          oprot.writeString(_iter14);
+        }
+        oprot.writeListEnd();
+      }
+      oprot.writeFieldEnd();
+    }
+    if (this.lintstring != null) {
+      field.name = "lintstring";
+      field.type = TType.LIST;
+      field.id = 5;
+      oprot.writeFieldBegin(field);
+      {
+        oprot.writeListBegin(new TList(TType.STRUCT, this.lintstring.size()));
+        for (IntString _iter15 : this.lintstring)        {
+          _iter15.write(oprot);
+        }
+        oprot.writeListEnd();
+      }
+      oprot.writeFieldEnd();
+    }
+    if (this.mstringstring != null) {
+      field.name = "mstringstring";
+      field.type = TType.MAP;
+      field.id = 6;
+      oprot.writeFieldBegin(field);
+      {
+        oprot.writeMapBegin(new TMap(TType.STRING, TType.STRING, this.mstringstring.size()));
+        for (String _iter16 : this.mstringstring.keySet())        {
+          oprot.writeString(_iter16);
+          oprot.writeString(this.mstringstring.get(_iter16));
+        }
+        oprot.writeMapEnd();
+      }
+      oprot.writeFieldEnd();
+    }
+    oprot.writeFieldStop();
+    oprot.writeStructEnd();
+  }
+
+  public String toString() {
+    StringBuilder sb = new StringBuilder("Complex(");
+    sb.append("aint:");
+    sb.append(this.aint);
+    sb.append(",astring:");
+    sb.append(this.astring);
+    sb.append(",lint:");
+    sb.append(this.lint);
+    sb.append(",lstring:");
+    sb.append(this.lstring);
+    sb.append(",lintstring:");
+    sb.append(this.lintstring);
+    sb.append(",mstringstring:");
+    sb.append(this.mstringstring);
+    sb.append(")");
+    return sb.toString();
+  }
+
+}
+

src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/Constants.java

@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ */
+package org.apache.hadoop.hive.serde2.thrift_test;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import com.facebook.thrift.*;
+
+public class Constants {
+
+}

src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/IntString.java

@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ */
+package org.apache.hadoop.hive.serde2.thrift_test;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import com.facebook.thrift.*;
+
+import com.facebook.thrift.protocol.*;
+import com.facebook.thrift.transport.*;
+
+public class IntString implements TBase, java.io.Serializable {
+  public int myint;
+  public String mystring;
+
+  public final Isset __isset = new Isset();
+  public static final class Isset implements java.io.Serializable {
+    public boolean myint = false;
+    public boolean mystring = false;
+  }
+
+  public IntString() {
+  }
+
+  public IntString(
+    int myint,
+    String mystring)
+  {
+    this();
+    this.myint = myint;
+    this.__isset.myint = true;
+    this.mystring = mystring;
+    this.__isset.mystring = true;
+  }
+
+  public boolean equals(Object that) {
+    if (that == null)
+      return false;
+    if (that instanceof IntString)
+      return this.equals((IntString)that);
+    return false;
+  }
+
+  public boolean equals(IntString that) {
+    if (that == null)
+      return false;
+
+    boolean this_present_myint = true;
+    boolean that_present_myint = true;
+    if (this_present_myint || that_present_myint) {
+      if (!(this_present_myint && that_present_myint))
+        return false;
+      if (this.myint != that.myint)
+        return false;
+    }
+
+    boolean this_present_mystring = true && (this.mystring != null);
+    boolean that_present_mystring = true && (that.mystring != null);
+    if (this_present_mystring || that_present_mystring) {
+      if (!(this_present_mystring && that_present_mystring))
+        return false;
+      if (!this.mystring.equals(that.mystring))
+        return false;
+    }
+
+    return true;
+  }
+
+  public int hashCode() {
+    return 0;
+  }
+
+  public void read(TProtocol iprot) throws TException {
+    TField field;
+    iprot.readStructBegin();
+    while (true)
+    {
+      field = iprot.readFieldBegin();
+      if (field.type == TType.STOP) { 
+        break;
+      }
+      switch (field.id)
+      {
+        case 1:
+          if (field.type == TType.I32) {
+            this.myint = iprot.readI32();
+            this.__isset.myint = true;
+          } else { 
+            TProtocolUtil.skip(iprot, field.type);
+          }
+          break;
+        case 2:
+          if (field.type == TType.STRING) {
+            this.mystring = iprot.readString();
+            this.__isset.mystring = true;
+          } else { 
+            TProtocolUtil.skip(iprot, field.type);
+          }
+          break;
+        default:
+          TProtocolUtil.skip(iprot, field.type);
+          break;
+      }
+      iprot.readFieldEnd();
+    }
+    iprot.readStructEnd();
+  }
+
+  public void write(TProtocol oprot) throws TException {
+    TStruct struct = new TStruct("IntString");
+    oprot.writeStructBegin(struct);
+    TField field = new TField();
+    field.name = "myint";
+    field.type = TType.I32;
+    field.id = 1;
+    oprot.writeFieldBegin(field);
+    oprot.writeI32(this.myint);
+    oprot.writeFieldEnd();
+    if (this.mystring != null) {
+      field.name = "mystring";
+      field.type = TType.STRING;
+      field.id = 2;
+      oprot.writeFieldBegin(field);
+      oprot.writeString(this.mystring);
+      oprot.writeFieldEnd();
+    }
+    oprot.writeFieldStop();
+    oprot.writeStructEnd();
+  }
+
+  public String toString() {
+    StringBuilder sb = new StringBuilder("IntString(");
+    sb.append("myint:");
+    sb.append(this.myint);
+    sb.append(",mystring:");
+    sb.append(this.mystring);
+    sb.append(")");
+    return sb.toString();
+  }
+
+}
+