HADOOP 4205. Fixed compilation problems.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@697293 13f79535-47bb-0310-9956-ffa450edef68

src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java

@@ -82,27 +82,30 @@ public class CliDriver {
         e.printStackTrace();
       }
 
-    } 
-    else {
+    } else {
       ret = qp.run(cmd);
       Vector<Vector<String>> res = new Vector<Vector<String>>();
-      while (qp.getResults(res)) {
-        SessionState ss  = SessionState.get();
-        PrintStream out = ss.out;
-
-        for (Vector<String> row:res)
+      while (qp.getResults(res))
+      {
+        SessionState ss = SessionState.get();
+        OutputStream out = ss.out;
+        try 
         {
-          boolean firstCol = true;
-          for (String col:row)
+
+          for (Vector<String> row:res)
           {
-            if (!firstCol)
+            for (String col:row)
+            {
+              out.write(col == null ? Utilities.nullStringOutput.getBytes() : col.getBytes());
               out.write(Utilities.tabCode);
-            out.print(col == null ? Utilities.nullStringOutput : col);
-            firstCol = false;
-          } 
-          out.write(Utilities.newLineCode);
+            } 
+            out.write(Utilities.newLineCode);
+          }
+          res.clear();
+
+        } catch (IOException e) {
+          e.printStackTrace();
         }
-        res.clear();
       }
     }
     return ret;
@@ -148,20 +151,16 @@ public class CliDriver {
     SessionState.initHiveLog4j();
 
     CliSessionState ss = new CliSessionState (new HiveConf(SessionState.class));
-    ss.in = System.in;
-    try {
-      ss.out = new PrintStream(System.out, true, "UTF-8");
-      ss.err = new PrintStream(System.err, true, "UTF-8");
-    } catch (UnsupportedEncodingException e) {
-      System.exit(3);
-    }
-
     SessionState.start(ss);
 
     if(! oproc.process_stage2(ss)) {
       System.exit(2);
     }
 
+    ss.in = System.in;
+    ss.out = System.out;
+    ss.err = System.err;
+
     sp = new SetProcessor();
     qp = new Driver();
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/io/HiveKey.java

@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.WritableComparator;
+
+/** HiveKey is a simple wrapper on Text which allows us to set the hashCode easily.
+ *  hashCode is used for hadoop partitioner.
+ */ 
+public class HiveKey extends BytesWritable {
+
+  private static final int LENGTH_BYTES = 4;
+  
+  boolean hashCodeValid;
+  public HiveKey() {
+    hashCodeValid = false; 
+  }
+  
+  protected int myHashCode; 
+  public void setHashCode(int myHashCode) {
+    this.hashCodeValid = true;
+    this.myHashCode = myHashCode;
+  }
+  public int hashCode() {
+    if (!hashCodeValid) {
+      throw new RuntimeException("Cannot get hashCode() from deserialized " + HiveKey.class);
+    }
+    return myHashCode;
+  }
+
+  /** A Comparator optimized for HiveKey. */ 
+  public static class Comparator extends WritableComparator {
+    public Comparator() {
+      super(HiveKey.class);
+    }
+    
+    /**
+     * Compare the buffers in serialized form.
+     */
+    public int compare(byte[] b1, int s1, int l1,
+                       byte[] b2, int s2, int l2) {
+      return compareBytes(b1, s1+LENGTH_BYTES, l1-LENGTH_BYTES, 
+                          b2, s2+LENGTH_BYTES, l2-LENGTH_BYTES);
+    }
+  }
+  
+  static {
+    WritableComparator.define(HiveKey.class, new Comparator());
+  }
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/ListTypeInfo.java

@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.typeinfo;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+
+/** A List Type has homogeneous elements.  All elements of the List has
+ *  the same TypeInfo which is returned by getListElementTypeInfo.
+ *  
+ *  Always use the TypeInfoFactory to create new TypeInfo objects, instead
+ *  of directly creating an instance of this class. 
+ */
+public class ListTypeInfo extends TypeInfo implements Serializable {
+
+  private static final long serialVersionUID = 1L;
+  TypeInfo listElementTypeInfo;
+  
+  /** For java serialization use only.
+   */
+  public ListTypeInfo() {}
+  
+  public String getTypeName() {
+    return org.apache.hadoop.hive.serde.Constants.LIST_TYPE_NAME 
+    + "<" + listElementTypeInfo.getTypeName() + ">";
+  }
+  
+  /** For java serialization use only.
+   */
+  public void setListElementTypeInfo(TypeInfo listElementTypeInfo) {
+    this.listElementTypeInfo = listElementTypeInfo;
+  }
+  
+  /** For TypeInfoFactory use only.
+   */
+  ListTypeInfo(TypeInfo elementTypeInfo) {
+    this.listElementTypeInfo = elementTypeInfo;
+  }
+
+  public Category getCategory() {
+    return Category.LIST;
+  }  
+
+  public TypeInfo getListElementTypeInfo() {
+    return listElementTypeInfo;
+  }
+
+  public boolean equals(Object other) {
+    if (this == other) return true;
+    if (!(other instanceof TypeInfo)) {
+      return false;
+    }
+    TypeInfo o = (TypeInfo) other;
+    return o.getCategory().equals(getCategory())
+        && o.getListElementTypeInfo().equals(getListElementTypeInfo());
+  }
+  
+  public int hashCode() {
+    return listElementTypeInfo.hashCode();
+  }
+  
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/MapTypeInfo.java

@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.typeinfo;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+
+/** A Map Type has homogeneous keys and homogeneous values.
+ *  All keys of the Map have the same TypeInfo, which is returned by
+ *  getMapKeyTypeInfo(); and all values of the Map has the same TypeInfo,
+ *  which is returned by getMapValueTypeInfo().
+ *  
+ *  Always use the TypeInfoFactory to create new TypeInfo objects, instead
+ *  of directly creating an instance of this class. 
+ */
+public class MapTypeInfo extends TypeInfo implements Serializable{
+
+  private static final long serialVersionUID = 1L;
+  
+  TypeInfo mapKeyTypeInfo;
+  TypeInfo mapValueTypeInfo;
+  
+  /** For java serialization use only.
+   */
+  public MapTypeInfo() {}
+  
+  public String getTypeName() {
+    return org.apache.hadoop.hive.serde.Constants.MAP_TYPE_NAME 
+    + "<" + mapKeyTypeInfo.getTypeName() + "," 
+    + mapValueTypeInfo.getTypeName() + ">";
+  }
+  
+  /** For java serialization use only.
+   */
+  public void setMapKeyTypeInfo(TypeInfo mapKeyTypeInfo) {
+    this.mapKeyTypeInfo = mapKeyTypeInfo;
+  }
+
+  /** For java serialization use only.
+   */
+  public void setMapValueTypeInfo(TypeInfo mapValueTypeInfo) {
+    this.mapValueTypeInfo = mapValueTypeInfo;
+  }
+  
+  // For TypeInfoFactory use only
+  MapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) {
+    this.mapKeyTypeInfo = keyTypeInfo;
+    this.mapValueTypeInfo = valueTypeInfo;
+  }
+
+  public Category getCategory() {
+    return Category.MAP;
+  }  
+
+  public TypeInfo getMapKeyTypeInfo() {
+    return mapKeyTypeInfo;
+  }
+
+  public TypeInfo getMapValueTypeInfo() {
+    return mapValueTypeInfo;
+  }
+
+  public boolean equals(Object other) {
+    if (this == other) return true;
+    if (!(other instanceof TypeInfo)) {
+      return false;
+    }
+    TypeInfo o = (TypeInfo) other;
+    return o.getCategory().equals(getCategory())
+        && o.getMapKeyTypeInfo().equals(getMapKeyTypeInfo())
+        && o.getMapValueTypeInfo().equals(getMapValueTypeInfo());
+  }
+  
+  public int hashCode() {
+    return mapKeyTypeInfo.hashCode() ^ mapValueTypeInfo.hashCode();
+  }
+  
+  
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/PrimitiveTypeInfo.java

@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.typeinfo;
+
+import java.io.Serializable;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+
+
+/** There are limited number of Primitive Types.
+ *  All Primitive Types are defined by TypeInfoFactory.isPrimitiveClass().
+ *  
+ *  Always use the TypeInfoFactory to create new TypeInfo objects, instead
+ *  of directly creating an instance of this class. 
+ */
+public class PrimitiveTypeInfo extends TypeInfo implements Serializable {
+
+  private static final long serialVersionUID = 1L;
+  
+  Class primitiveClass;
+  
+  /** For java serialization use only.
+   */
+  public PrimitiveTypeInfo() {}
+
+  public String getTypeName() {
+    return ObjectInspectorUtils.getClassShortName(primitiveClass.getName());
+  }
+  
+  
+  /** For java serialization use only.
+   */
+  public void setPrimitiveClass(Class primitiveClass) {
+    this.primitiveClass = primitiveClass;
+  }
+  
+  /** For TypeInfoFactory use only.
+   */
+  PrimitiveTypeInfo(Class primitiveClass) {
+    this.primitiveClass = primitiveClass;
+  }
+  
+  public Category getCategory() {
+    return Category.PRIMITIVE;
+  }
+
+  public Class getPrimitiveClass() {
+    return primitiveClass;
+  }
+
+  public boolean equals(Object other) {
+    if (this == other) return true;
+    if (!(other instanceof TypeInfo)) {
+      return false;
+    }
+    TypeInfo o = (TypeInfo) other;
+    return o.getCategory().equals(getCategory())
+        && o.getPrimitiveClass().equals(getPrimitiveClass());
+  }
+  
+  public int hashCode() {
+    return primitiveClass.hashCode();
+  }
+  
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/StructTypeInfo.java

@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.typeinfo;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+
+/** StructTypeInfo represents the TypeInfo of a struct.
+ *  A struct contains one or more fields each of which has a unique name
+ *  and its own TypeInfo.  Different fields can have the same or different
+ *  TypeInfo. 
+ *  
+ *  Always use the TypeInfoFactory to create new TypeInfo objects, instead
+ *  of directly creating an instance of this class. 
+ */
+public class StructTypeInfo extends TypeInfo implements Serializable{
+
+  private static final long serialVersionUID = 1L;
+  
+  ArrayList<String> allStructFieldNames;
+  ArrayList<TypeInfo> allStructFieldTypeInfos;
+  
+  /** For java serialization use only.
+   */
+  public StructTypeInfo() {}
+
+  public String getTypeName() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("struct{");
+    for(int i=0; i<allStructFieldNames.size(); i++) {
+      if (i>0) sb.append(",");
+      sb.append(allStructFieldNames.get(i));
+      sb.append(":");
+      sb.append(allStructFieldTypeInfos.get(i).getTypeName());
+    }
+    sb.append("}");
+    return sb.toString();
+  }
+  
+  /** For java serialization use only.
+   */
+  public void setAllStructFieldNames(ArrayList<String> allStructFieldNames) {
+    this.allStructFieldNames = allStructFieldNames;
+  }
+
+  /** For java serialization use only.
+   */
+  public void setAllStructFieldTypeInfos(
+      ArrayList<TypeInfo> allStructFieldTypeInfos) {
+    this.allStructFieldTypeInfos = allStructFieldTypeInfos;
+  }
+  
+  /** For TypeInfoFactory use only.
+   */
+  StructTypeInfo(List<String> names, List<TypeInfo> typeInfos) {
+    assert(allStructFieldNames.size() == typeInfos.size());
+    allStructFieldNames = new ArrayList<String>();
+    allStructFieldNames.addAll(names);
+    allStructFieldTypeInfos = new ArrayList<TypeInfo>();
+    allStructFieldTypeInfos.addAll(typeInfos);
+  }
+
+  public Category getCategory() {
+    return Category.STRUCT;
+  }
+  
+  public List<String> getAllStructFieldNames() {
+    return java.util.Collections.unmodifiableList(allStructFieldNames);
+  }
+  
+  public List<TypeInfo> getAllStructFieldTypeInfos() {
+    return java.util.Collections.unmodifiableList(allStructFieldTypeInfos);
+  }
+  
+  public TypeInfo getStructFieldTypeInfo(String field) {
+    for(int i=0; i<allStructFieldNames.size(); i++) {
+      if (field.equals(allStructFieldNames.get(i))) {
+        return allStructFieldTypeInfos.get(i);
+      }
+    }
+    throw new RuntimeException("cannot find field " + field + " in " + allStructFieldNames);
+    // return null;
+  }
+  
+  @Override
+  public boolean equals(Object other) {
+    if (this == other) return true;
+    if (!(other instanceof TypeInfo)) {
+      return false;
+    }
+    TypeInfo o = (TypeInfo) other;
+    return o.getCategory().equals(getCategory())
+        && o.getAllStructFieldNames().equals(getAllStructFieldNames())
+        && o.getAllStructFieldTypeInfos().equals(getAllStructFieldTypeInfos());
+  }
+  
+  public int hashCode() {
+    return allStructFieldNames.hashCode() ^ allStructFieldTypeInfos.hashCode();
+  }
+  
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/TypeInfo.java

@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.typeinfo;
+
+import java.io.Serializable;import java.util.List;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+
+/**
+ * Stores information about a type (DDL).
+ * Always use the TypeInfoFactory to create new TypeInfo objects.
+ * 
+ * We support 4 categories of types:
+ * 1. Primitive objects (String, Number, etc)
+ * 2. List objects (a list of objects of a single type)
+ * 3. Map objects (a map from objects of one type to objects of another type)
+ * 4. Struct objects (a list of fields with names and their own types)
+ */
+public abstract class TypeInfo implements Serializable {
+
+  protected TypeInfo() {}
+  
+  public String getTypeName() {
+    throw new RuntimeException("Unsupported: " + this.getClass() + ".getCategory()");
+  }
+  
+  public Category getCategory() {
+    throw new RuntimeException("Unsupported: " + this.getClass() + ".getCategory()");
+  }
+  
+  public Class<?> getPrimitiveClass() {
+    throw new RuntimeException("Unsupported: " + this.getClass() + ".getPrimitiveClass()");
+  }
+  
+  public TypeInfo getListElementTypeInfo() {
+    throw new RuntimeException("Unsupported: " + this.getClass() + ".getListElementTypeInfo()");
+  }
+  
+  public TypeInfo getMapKeyTypeInfo() {
+    throw new RuntimeException("Unsupported: " + this.getClass() + ".getMapKeyTypeInfo()");
+  }
+  public TypeInfo getMapValueTypeInfo() {
+    throw new RuntimeException("Unsupported: " + this.getClass() + ".getMapValueTypeInfo()");
+  }
+  
+  public List<String> getAllStructFieldNames() {
+    throw new RuntimeException("Unsupported: " + this.getClass() + ".getAllStructFieldNames()");
+  }
+  public List<TypeInfo> getAllStructFieldTypeInfos() {
+    throw new RuntimeException("Unsupported: " + this.getClass() + ".getAllStructFieldTypeInfos()");
+  }
+  public TypeInfo getStructFieldTypeInfo(String field) {
+    throw new RuntimeException("Unsupported: " + this.getClass() + ".getStructFieldTypeInfo()");
+  }
+  
+  public String toString() {
+    return getTypeName();
+  }
+
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/TypeInfoFactory.java

@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.typeinfo;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+
+/**
+ * TypeInfoFactory can be used to create the TypeInfo object for any types.
+ * 
+ * TypeInfo objects are all read-only so we can reuse them easily. TypeInfoFactory 
+ * has internal cache to make sure we don't create 2 TypeInfo objects that represents the
+ * same type.
+ */
+public class TypeInfoFactory {
+
+  static HashMap<Class<?>, TypeInfo> cachedPrimitiveTypeInfo = new HashMap<Class<?>, TypeInfo>();
+  public static TypeInfo getPrimitiveTypeInfo(Class<?> primitiveClass) {
+    assert(ObjectInspectorUtils.isPrimitiveClass(primitiveClass));
+    primitiveClass = ObjectInspectorUtils.generalizePrimitive(primitiveClass);
+    TypeInfo result = cachedPrimitiveTypeInfo.get(primitiveClass);
+    if (result == null) { 
+      result = new PrimitiveTypeInfo(primitiveClass);
+      cachedPrimitiveTypeInfo.put(primitiveClass, result);
+    }
+    return result;
+  }
+  
+  static HashMap<ArrayList<List<?>>, TypeInfo> cachedStructTypeInfo = new HashMap<ArrayList<List<?>>, TypeInfo>();
+  public static TypeInfo getStructTypeInfo(List<String> names, List<TypeInfo> typeInfos) {
+    ArrayList<List<?>> signature = new ArrayList<List<?>>(2);
+    signature.add(names);
+    signature.add(typeInfos);
+    TypeInfo result = cachedStructTypeInfo.get(signature);
+    if (result == null) { 
+      result = new StructTypeInfo(names, typeInfos);
+      cachedStructTypeInfo.put(signature, result);
+    }
+    return result;
+  }
+  
+  static HashMap<TypeInfo, TypeInfo> cachedListTypeInfo = new HashMap<TypeInfo, TypeInfo>();
+  public static TypeInfo getListTypeInfo(TypeInfo elementTypeInfo) {
+    TypeInfo result = cachedListTypeInfo.get(elementTypeInfo);
+    if (result == null) { 
+      result = new ListTypeInfo(elementTypeInfo);
+      cachedListTypeInfo.put(elementTypeInfo, result);
+    }
+    return result;
+  }
+
+  static HashMap<ArrayList<TypeInfo>, TypeInfo> cachedMapTypeInfo = new HashMap<ArrayList<TypeInfo>, TypeInfo>();
+  public static TypeInfo getMapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) {
+    ArrayList<TypeInfo> signature = new ArrayList<TypeInfo>(2);
+    signature.add(keyTypeInfo);
+    signature.add(valueTypeInfo);
+    TypeInfo result = cachedMapTypeInfo.get(signature);
+    if (result == null) { 
+      result = new MapTypeInfo(keyTypeInfo, valueTypeInfo);
+      cachedMapTypeInfo.put(signature, result);
+    }
+    return result;
+  }
+  
+  
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/TypeInfoUtils.java

@@ -0,0 +1,112 @@
+package org.apache.hadoop.hive.ql.typeinfo;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
+public class TypeInfoUtils {
+  
+  static HashMap<TypeInfo, ObjectInspector> cachedStandardObjectInspector = new HashMap<TypeInfo, ObjectInspector>();
+  /**
+   * Returns the standard object inspector that can be used to translate an object of that typeInfo
+   * to a standard object type.  
+   */
+  public static ObjectInspector getStandardObjectInspectorFromTypeInfo(TypeInfo typeInfo) {
+    ObjectInspector result = cachedStandardObjectInspector.get(typeInfo);
+    if (result == null) {
+      switch(typeInfo.getCategory()) {
+        case PRIMITIVE: {
+          result = ObjectInspectorFactory.getStandardPrimitiveObjectInspector(typeInfo.getPrimitiveClass());
+          break;
+        }
+        case LIST: {
+          ObjectInspector elementObjectInspector = getStandardObjectInspectorFromTypeInfo(typeInfo.getListElementTypeInfo());
+          result = ObjectInspectorFactory.getStandardListObjectInspector(elementObjectInspector);
+          break;
+        }
+        case MAP: {
+          ObjectInspector keyObjectInspector = getStandardObjectInspectorFromTypeInfo(typeInfo.getMapKeyTypeInfo());
+          ObjectInspector valueObjectInspector = getStandardObjectInspectorFromTypeInfo(typeInfo.getMapValueTypeInfo());
+          result = ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector);
+          break;
+        }
+        case STRUCT: {
+          List<String> fieldNames = typeInfo.getAllStructFieldNames();
+          List<TypeInfo> fieldTypeInfos = typeInfo.getAllStructFieldTypeInfos();
+          List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fieldTypeInfos.size());
+          for(int i=0; i<fieldTypeInfos.size(); i++) {
+            fieldObjectInspectors.add(getStandardObjectInspectorFromTypeInfo(fieldTypeInfos.get(i)));
+          }
+          result = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
+          break;
+        }
+        default: {
+          result = null;
+        }
+      }
+      cachedStandardObjectInspector.put(typeInfo, result);
+    }
+    return result;
+  }
+
+  
+  /**
+   * Get the TypeInfo object from the ObjectInspector object by recursively going into the
+   * ObjectInspector structure.
+   */
+  public static TypeInfo getTypeInfoFromObjectInspector(ObjectInspector oi) {
+//    OPTIMIZATION for later.
+//    if (oi instanceof TypeInfoBasedObjectInspector) {
+//      TypeInfoBasedObjectInspector typeInfoBasedObjectInspector = (ObjectInspector)oi;
+//      return typeInfoBasedObjectInspector.getTypeInfo();
+//    }
+    
+    // Recursively going into ObjectInspector structure
+    TypeInfo result = null;
+    switch (oi.getCategory()) {
+      case PRIMITIVE: {
+        PrimitiveObjectInspector poi =(PrimitiveObjectInspector)oi;
+        result = TypeInfoFactory.getPrimitiveTypeInfo(poi.getPrimitiveClass());
+        break;
+      }
+      case LIST: {
+        ListObjectInspector loi = (ListObjectInspector)oi;
+        result = TypeInfoFactory.getListTypeInfo(
+            getTypeInfoFromObjectInspector(loi.getListElementObjectInspector()));
+        break;
+      }
+      case MAP: {
+        MapObjectInspector moi = (MapObjectInspector)oi;
+        result = TypeInfoFactory.getMapTypeInfo(
+            getTypeInfoFromObjectInspector(moi.getMapKeyObjectInspector()),
+            getTypeInfoFromObjectInspector(moi.getMapValueObjectInspector()));
+        break;
+      }
+      case STRUCT: {
+        StructObjectInspector soi = (StructObjectInspector)oi;
+        List<? extends StructField> fields = soi.getAllStructFieldRefs();
+        List<String> fieldNames = new ArrayList<String>(fields.size());
+        List<TypeInfo> fieldTypeInfos = new ArrayList<TypeInfo>(fields.size());
+        for(StructField f : fields) {
+          fieldNames.add(f.getFieldName());
+          fieldTypeInfos.add(getTypeInfoFromObjectInspector(f.getFieldObjectInspector()));
+        }
+        result = TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos);
+        break;
+      }
+      default: {
+        throw new RuntimeException("Unknown ObjectInspector category!");
+      }
+    }
+    return result;
+  }
+    
+}

src/contrib/hive/ql/src/test/queries/clientpositive/inputddl4.q

@@ -0,0 +1,10 @@
+-- a simple test to test sorted/clustered syntax
+CREATE TABLE INPUTDDL4(viewTime DATETIME, userid INT,
+                       page_url STRING, referrer_url STRING, 
+                       friends ARRAY<BIGINT>, properties MAP<STRING, STRING>,
+                       ip STRING COMMENT 'IP Address of the User') 
+    COMMENT 'This is the page view table' 
+    PARTITIONED BY(ds DATETIME, country STRING) 
+    CLUSTERED BY(userid) SORTED BY(viewTime) INTO 32 BUCKETS;
+DESCRIBE INPUTDDL4;
+DROP TABLE INPUTDDL4;

src/contrib/hive/ql/src/test/queries/clientpositive/inputddl5.q

@@ -0,0 +1,8 @@
+-- test for internationalization
+-- kv4.txt contains the utf-8 character 0xE982B5E993AE which we are verifying later on
+CREATE TABLE INPUTDDL5(name STRING);
+LOAD DATA LOCAL INPATH '../data/files/kv4.txt' INTO TABLE INPUTDDL5;
+DESCRIBE INPUTDDL5;
+SELECT INPUTDDL5.name from INPUTDDL5;
+SELECT count(1) FROM INPUTDDL5 WHERE INPUTDDL5.name = _UTF-8 0xE982B5E993AE;
+DROP TABLE INPUTDDL5;

src/contrib/hive/ql/src/test/results/clientpositive/inputddl4.q.out

@@ -0,0 +1,9 @@
+viewtime	datetime
+userid	int
+page_url	string
+referrer_url	string
+friends	array<bigint>
+properties	map<string,string>
+ip	string	'IP Address of the User'
+ds	datetime
+country	string

src/contrib/hive/ql/src/test/results/clientpositive/inputddl5.q.out

@@ -0,0 +1,5 @@
+name	string
+邵铮
+
+1
+