HADOOP-4230. Fix for serde2 interface, limit operator, select * operator,

UDF trim functions and sampling. (Ashish Thusoo via dhruba)



git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@706704 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -1006,6 +1006,9 @@ Release 0.19.0 - Unreleased
     particular mapred.tasktracker.tasks.maxmemory and mapred.task.max.memory.
     (Hemanth Yamijala via omalley)
 
+    HADOOP-4230. Fix for serde2 interface, limit operator, select * operator,
+    UDF trim functions and sampling. (Ashish Thusoo via dhruba)
+
 Release 0.18.2 - Unreleased
 
   BUG FIXES

src/contrib/hive/bin/hive

@@ -48,19 +48,36 @@ done
 # add the auxillary jars such as serdes
 if [ -d "${HIVE_AUX_JARS_PATH}" ]; then
   for f in ${HIVE_AUX_JARS_PATH}/*.jar; do
-    AUX_CLASSPATH=${AUX_CLASSPATH}:$f;
+    if [[ ! -f $f ]]; then
+        continue;
+    fi
+    AUX_CLASSPATH=${AUX_CLASSPATH}:$f
     if [ "${AUX_PARAM}" == "" ]; then
-        AUX_PARAM=$f
+        AUX_PARAM=file://$f
     else
-        AUX_PARAM=${AUX_PARAM},$f;
+        AUX_PARAM=${AUX_PARAM},file://$f;
     fi
   done
-else
+elif [ "${HIVE_AUX_JARS_PATH}" != "" ]; then 
   AUX_CLASSPATH=${HIVE_AUX_JARS_PATH}
-  AUX_PARAM=${HIVE_AUX_JARS_PATH}
+  AUX_PARAM=file://${HIVE_AUX_JARS_PATH}
+  AUX_PARAM=`echo $AUX_PARAM | sed 's/,/,file:\/\//g'`
 fi
-CLASSPATH=${CLASSPATH}:${AUX_CLASSPATH}
 
+# adding jars from auxlib directory
+for f in ${HIVE_HOME}/auxlib/*.jar; do
+  if [[ ! -f $f ]]; then
+      continue;
+  fi
+
+  AUX_CLASSPATH=${AUX_CLASSPATH}:$f
+  if [ "${AUX_PARAM}" == "" ]; then
+    AUX_PARAM=file://$f
+  else
+    AUX_PARAM=${AUX_PARAM},file://$f;
+  fi
+done
+CLASSPATH=${CLASSPATH}:${AUX_CLASSPATH}
 
 # pass classpath to hadoop
 export HADOOP_CLASSPATH=${CLASSPATH}
@@ -79,23 +96,24 @@ fi
 
 HADOOP=$HADOOP_HOME/bin/hadoop
 if [ ! -f ${HADOOP} ]; then
-  echo "Cannot find hadoop installation: \"$HADOOP\" does not exist";
+  echo "Cannot find hadoop installation: \$HADOOP_HOME must be set or hadoop must be in the path";
   exit 4;
 fi
 
 # override default value of hadoop.bin.path to point to what we are running off
 # if the user specified a -D override - this will be further overriden
-export HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.bin.path=$HADOOP"
+export HIVE_OPTS="-hiveconf hadoop.bin.path=$HADOOP"
 
 CLASS=org.apache.hadoop.hive.cli.CliDriver
 
 if [ "${AUX_PARAM}" != "" ]; then
-  HADOOP_OPTS="-Dhive.aux.jars.path=${AUX_PARAM}"
+  HIVE_OPTS="$HIVE_OPTS -hiveconf hive.aux.jars.path=${AUX_PARAM}"
   AUX_JARS_CMD_LINE="-libjars ${AUX_PARAM}"
 fi
 #echo "CLASSPATH="$CLASSPATH
 #echo "AUX_LIB_PATH"=$AUX_PARAM
 #echo "HADOOP_OPTS="$HADOOP_OPTS
 # note: $@ only works in "$@"
-exec $HADOOP jar $AUX_JARS_CMD_LINE ${HIVE_LIB}/hive_cli.jar $CLASS "$@"
+#echo $HADOOP jar $AUX_JARS_CMD_LINE ${HIVE_LIB}/hive_cli.jar $CLASS $HIVE_OPTS "$@"
+exec $HADOOP jar $AUX_JARS_CMD_LINE ${HIVE_LIB}/hive_cli.jar $CLASS $HIVE_OPTS "$@"
 

src/contrib/hive/build-common.xml

@@ -53,6 +53,7 @@
   <property name="test.build.classes" value="${test.build.dir}/classes"/>
   <property name="test.build.javadoc" value="${test.build.dir}/docs/api"/>
   <property name="test.include" value="Test*"/>
+  <property name="test.exclude" value="TestSerDe"/>
   <property name="test.classpath.id" value="test.classpath"/>
   <property name="test.output" value="true"/>
   <property name="test.timeout" value="900000"/>
@@ -64,6 +65,7 @@
     <pathelement location="${hadoop.conf.dir}"/>
     <pathelement location="${test.data.dir}/conf"/>
     <pathelement location="${hive.conf.dir}"/>
+    <fileset dir="${test.src.data.dir}" includes="files/*.jar"/>
     <path refid="classpath"/>
   </path>
 
@@ -95,6 +97,7 @@
     <pathelement location="${hadoop.root}/build/contrib/hive/metastore/classes"/>
     <pathelement location="${hadoop.root}/build/contrib/hive/ql/classes"/>
     <pathelement location="${hadoop.root}/build/contrib/hive/cli/classes"/>
+    <fileset dir="${test.src.data.dir}" includes="files/*.jar"/>
     <fileset dir="${basedir}" includes="lib/*.jar"/>
     <path refid="common-classpath"/>
   </path>
@@ -215,7 +218,7 @@
       <sysproperty key="test.output.overwrite" value="${overwrite}"/>
       <sysproperty key="log4j.configuration" value="file://${test.data.dir}/conf/hive-log4j.properties"/>
       <sysproperty key="derby.stream.error.file" value="${test.build.dir}/derby.log"/>
-      <sysproperty key="hive.aux.jars.path" value="file://${test.build.dir}/test-udfs.jar"/>
+      <sysproperty key="hive.aux.jars.path" value="${test.build.dir}/test-udfs.jar,${test.src.data.dir}/files/TestSerDe.jar"/>
       <classpath refid="${test.classpath.id}"/>
       <formatter type="${test.junit.output.format}" />
       <batchtest todir="${test.build.dir}" unless="testcase">

src/contrib/hive/build.xml

@@ -140,9 +140,11 @@
     <copy file="${basedir}/conf/hive-log4j.properties" todir="${target.conf.dir}"/>
     <copy file="${basedir}/conf/jpox.properties" todir="${target.conf.dir}"/>
     <copy todir="${target.lib.dir}/php">
+      <fileset dir="${hive.root}/serde/src/gen-php"/>
       <fileset dir="${hive.root}/metastore/src/gen-php"/>
     </copy>
     <copy todir="${target.lib.dir}/py">
+      <fileset dir="${hive.root}/serde/src/gen-py"/>
       <fileset dir="${hive.root}/metastore/src/gen-py"/>
     </copy>
     <copy todir="${target.lib.dir}" preservelastmodified="true" flatten="true">

src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java

@@ -23,6 +23,7 @@ import jline.*;
 import java.io.*;
 import java.util.*;
 
+import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.Utilities.StreamPrinter;
@@ -35,9 +36,11 @@ import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
 public class CliDriver {
 
   public final static String prompt = "hive";
+  public final static String prompt2 = "    "; // when ';' is not yet seen
 
   public static SetProcessor sp;
   public static Driver qp;
+  public static FsShell dfs;
 
   public CliDriver(CliSessionState ss) {
     SessionState.start(ss);
@@ -81,30 +84,46 @@ public class CliDriver {
       catch (Exception e) {
         e.printStackTrace();
       }
-
-    } 
-    else {
+    } else if (cmd.startsWith("dfs")) {
+      // dfs shell commands
+      SessionState ss = SessionState.get();
+      if(dfs == null)
+        dfs = new FsShell(ss.getConf());
+      String hadoopCmd = cmd.replaceFirst("dfs\\s+", "");
+      hadoopCmd = hadoopCmd.trim();
+      if (hadoopCmd.endsWith(";")) {
+        hadoopCmd = hadoopCmd.substring(0, hadoopCmd.length()-1);
+      }
+      String[] args = hadoopCmd.split("\\s+");
+      try {
+        PrintStream oldOut = System.out;
+        System.setOut(ss.out);
+        int val = dfs.run(args);
+        System.setOut(oldOut);
+        if (val != 0) {
+          ss.err.write((new String("Command failed with exit code = " + val)).getBytes());
+        }
+      } catch (Exception e) {
+        ss.err.println("Exception raised from DFSShell.run " + e.getLocalizedMessage()); 
+      }
+    } else {
       ret = qp.run(cmd);
-      Vector<Vector<String>> res = new Vector<Vector<String>>();
+      Vector<String> res = new Vector<String>();
       while (qp.getResults(res)) {
-        SessionState ss  = SessionState.get();
-        PrintStream out = ss.out;
-
-        for (Vector<String> row:res)
-        {
-          boolean firstCol = true;
-          for (String col:row)
-          {
-            if (!firstCol)
-              out.write(Utilities.tabCode);
-            out.print(col == null ? Utilities.nullStringOutput : col);
-            firstCol = false;
-          } 
-          out.write(Utilities.newLineCode);
-        }
+      	for (String r:res) {
+          SessionState ss  = SessionState.get();
+          PrintStream out = ss.out;
+          out.println(r);
+      	}
         res.clear();
       }
+
+      int cret = qp.close();
+      if (ret == 0) {
+        ret = cret;
+      }
     }
+
     return ret;
   }
 
@@ -162,8 +181,15 @@ public class CliDriver {
       System.exit(2);
     }
 
+    // set all properties specified via command line
+    HiveConf conf = ss.getConf();
+    for(Map.Entry<Object, Object> item: ss.cmdProperties.entrySet()) {
+      conf.set((String) item.getKey(), (String) item.getValue());
+    }
+
     sp = new SetProcessor();
     qp = new Driver();
+    dfs = new FsShell(ss.getConf());
 
     if(ss.execString != null) {
       System.exit(processLine(ss.execString));
@@ -199,9 +225,20 @@ public class CliDriver {
     int ret = 0;
     Log LOG = LogFactory.getLog("CliDriver");
     LogHelper console = new LogHelper(LOG);
-    while ((line = reader.readLine(prompt+"> ")) != null) {
+    String prefix = "";
+    String curPrompt = prompt;
+    while ((line = reader.readLine(curPrompt+"> ")) != null) {
       long start = System.currentTimeMillis();
-      ret = processLine(line);
+      if(line.trim().endsWith(";")) {
+        line = prefix + " " + line;
+        ret = processLine(line);
+        prefix = "";
+        curPrompt = prompt;
+      } else {
+        prefix = prefix + line;
+        curPrompt = prompt2;
+        continue;
+      }
       long end = System.currentTimeMillis();
       if (end > start) {
         double timeTaken = (double)(end-start)/1000.0;

src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.hive.cli;
 
+import java.util.Properties;
+
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -33,6 +35,11 @@ public class CliSessionState extends SessionState {
    */
   public String fileName;
 
+  /**
+   * properties set from -hiveconf via cmdline
+   */
+  public Properties cmdProperties = new Properties();
+
 
   public CliSessionState() {
     super();

src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java

@@ -200,6 +200,15 @@ public class OptionsProcessor {
       printUsage("-e and -f option cannot be specified simultaneously");
       return false;
     }
+
+    List<String> hiveConfArgs = (List<String>)cmdLine.getValue(confOptions); 
+    if (null != hiveConfArgs){
+      for(String s : hiveConfArgs){
+        String []parts = s.split("=", 2); 
+        ss.cmdProperties.setProperty(parts[0], parts[1]);
+      }
+    }
+
     return true;
   }
 

src/contrib/hive/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java

@@ -80,6 +80,7 @@ public class HiveConf extends Configuration {
     // for hive script operator
     HIVETABLENAME("hive.table.name", ""),
     HIVEPARTITIONNAME("hive.partition.name", ""),
+    HIVEPARTITIONPRUNER("hive.partition.pruning", "nonstrict"),
     HIVEALIAS("hive.alias", "");
     
     public final String varname;
@@ -170,13 +171,13 @@ public class HiveConf extends Configuration {
     // let's add the hive configuration 
     URL hconfurl = getClassLoader().getResource("hive-default.xml");
     if(hconfurl == null) {
-      l4j.warn("Unable to locate default hive configuration");
+      l4j.debug("hive-default.xml not found.");
     } else {
       addResource(hconfurl);
     }
     URL hsiteurl = getClassLoader().getResource("hive-site.xml");
     if(hsiteurl == null) {
-      l4j.warn("Unable to locate hive site configuration");
+      l4j.debug("hive-site.xml not found.");
     } else {
       addResource(hsiteurl);
     }

src/contrib/hive/conf/hive-default.xml

@@ -9,6 +9,21 @@
 <!-- users do not have to edit hadoop configuration files (that may be managed as a centralized -->
 <!-- resource).                                                                                 -->
 
+<!-- Hadoop Setup -->
+<property>
+  <name>hadoop.bin.path</name>
+  <value>${user.dir}/../../../bin/hadoop</value>
+  <!-- note that the hive shell script also uses this property name -->
+  <description>Path to hadoop binary. Assumes that by default we are executing from hive</description>
+</property>
+
+<property>
+  <name>hadoop.config.dir</name>
+  <value>${user.dir}/../../../conf</value>
+  <!-- note that the hive shell script also uses this property name -->
+  <description>Path to hadoop configuration. Again assumes that by default we are executing from hive/</description>
+</property>
+
 <!-- Hive Execution Parameters -->
 <property>
   <name>hive.exec.scratchdir</name>

src/contrib/hive/data/files/kv1_cb.txt

@@ -0,0 +1,500 @@
+238val_238
+86val_86
+311val_311
+27val_27
+165val_165
+409val_409
+255val_255
+278val_278
+98val_98
+484val_484
+265val_265
+193val_193
+401val_401
+150val_150
+273val_273
+224val_224
+369val_369
+66val_66
+128val_128
+213val_213
+146val_146
+406val_406
+429val_429
+374val_374
+152val_152
+469val_469
+145val_145
+495val_495
+37val_37
+327val_327
+281val_281
+277val_277
+209val_209
+15val_15
+82val_82
+403val_403
+166val_166
+417val_417
+430val_430
+252val_252
+292val_292
+219val_219
+287val_287
+153val_153
+193val_193
+338val_338
+446val_446
+459val_459
+394val_394
+237val_237
+482val_482
+174val_174
+413val_413
+494val_494
+207val_207
+199val_199
+466val_466
+208val_208
+174val_174
+399val_399
+396val_396
+247val_247
+417val_417
+489val_489
+162val_162
+377val_377
+397val_397
+309val_309
+365val_365
+266val_266
+439val_439
+342val_342
+367val_367
+325val_325
+167val_167
+195val_195
+475val_475
+17val_17
+113val_113
+155val_155
+203val_203
+339val_339
+0val_0
+455val_455
+128val_128
+311val_311
+316val_316
+57val_57
+302val_302
+205val_205
+149val_149
+438val_438
+345val_345
+129val_129
+170val_170
+20val_20
+489val_489
+157val_157
+378val_378
+221val_221
+92val_92
+111val_111
+47val_47
+72val_72
+4val_4
+280val_280
+35val_35
+427val_427
+277val_277
+208val_208
+356val_356
+399val_399
+169val_169
+382val_382
+498val_498
+125val_125
+386val_386
+437val_437
+469val_469
+192val_192
+286val_286
+187val_187
+176val_176
+54val_54
+459val_459
+51val_51
+138val_138
+103val_103
+239val_239
+213val_213
+216val_216
+430val_430
+278val_278
+176val_176
+289val_289
+221val_221
+65val_65
+318val_318
+332val_332
+311val_311
+275val_275
+137val_137
+241val_241
+83val_83
+333val_333
+180val_180
+284val_284
+12val_12
+230val_230
+181val_181
+67val_67
+260val_260
+404val_404
+384val_384
+489val_489
+353val_353
+373val_373
+272val_272
+138val_138
+217val_217
+84val_84
+348val_348
+466val_466
+58val_58
+8val_8
+411val_411
+230val_230
+208val_208
+348val_348
+24val_24
+463val_463
+431val_431
+179val_179
+172val_172
+42val_42
+129val_129
+158val_158
+119val_119
+496val_496
+0val_0
+322val_322
+197val_197
+468val_468
+393val_393
+454val_454
+100val_100
+298val_298
+199val_199
+191val_191
+418val_418
+96val_96
+26val_26
+165val_165
+327val_327
+230val_230
+205val_205
+120val_120
+131val_131
+51val_51
+404val_404
+43val_43
+436val_436
+156val_156
+469val_469
+468val_468
+308val_308
+95val_95
+196val_196
+288val_288
+481val_481
+457val_457
+98val_98
+282val_282
+197val_197
+187val_187
+318val_318
+318val_318
+409val_409
+470val_470
+137val_137
+369val_369
+316val_316
+169val_169
+413val_413
+85val_85
+77val_77
+0val_0
+490val_490
+87val_87
+364val_364
+179val_179
+118val_118
+134val_134
+395val_395
+282val_282
+138val_138
+238val_238
+419val_419
+15val_15
+118val_118
+72val_72
+90val_90
+307val_307
+19val_19
+435val_435
+10val_10
+277val_277
+273val_273
+306val_306
+224val_224
+309val_309
+389val_389
+327val_327
+242val_242
+369val_369
+392val_392
+272val_272
+331val_331
+401val_401
+242val_242
+452val_452
+177val_177
+226val_226
+5val_5
+497val_497
+402val_402
+396val_396
+317val_317
+395val_395
+58val_58
+35val_35
+336val_336
+95val_95
+11val_11
+168val_168
+34val_34
+229val_229
+233val_233
+143val_143
+472val_472
+322val_322
+498val_498
+160val_160
+195val_195
+42val_42
+321val_321
+430val_430
+119val_119
+489val_489
+458val_458
+78val_78
+76val_76
+41val_41
+223val_223
+492val_492
+149val_149
+449val_449
+218val_218
+228val_228
+138val_138
+453val_453
+30val_30
+209val_209
+64val_64
+468val_468
+76val_76
+74val_74
+342val_342
+69val_69
+230val_230
+33val_33
+368val_368
+103val_103
+296val_296
+113val_113
+216val_216
+367val_367
+344val_344
+167val_167
+274val_274
+219val_219
+239val_239
+485val_485
+116val_116
+223val_223
+256val_256
+263val_263
+70val_70
+487val_487
+480val_480
+401val_401
+288val_288
+191val_191
+5val_5
+244val_244
+438val_438
+128val_128
+467val_467
+432val_432
+202val_202
+316val_316
+229val_229
+469val_469
+463val_463
+280val_280
+2val_2
+35val_35
+283val_283
+331val_331
+235val_235
+80val_80
+44val_44
+193val_193
+321val_321
+335val_335
+104val_104
+466val_466
+366val_366
+175val_175
+403val_403
+483val_483
+53val_53
+105val_105
+257val_257
+406val_406
+409val_409
+190val_190
+406val_406
+401val_401
+114val_114
+258val_258
+90val_90
+203val_203
+262val_262
+348val_348
+424val_424
+12val_12
+396val_396
+201val_201
+217val_217
+164val_164
+431val_431
+454val_454
+478val_478
+298val_298
+125val_125
+431val_431
+164val_164
+424val_424
+187val_187
+382val_382
+5val_5
+70val_70
+397val_397
+480val_480
+291val_291
+24val_24
+351val_351
+255val_255
+104val_104
+70val_70
+163val_163
+438val_438
+119val_119
+414val_414
+200val_200
+491val_491
+237val_237
+439val_439
+360val_360
+248val_248
+479val_479
+305val_305
+417val_417
+199val_199
+444val_444
+120val_120
+429val_429
+169val_169
+443val_443
+323val_323
+325val_325
+277val_277
+230val_230
+478val_478
+178val_178
+468val_468
+310val_310
+317val_317
+333val_333
+493val_493
+460val_460
+207val_207
+249val_249
+265val_265
+480val_480
+83val_83
+136val_136
+353val_353
+172val_172
+214val_214
+462val_462
+233val_233
+406val_406
+133val_133
+175val_175
+189val_189
+454val_454
+375val_375
+401val_401
+421val_421
+407val_407
+384val_384
+256val_256
+26val_26
+134val_134
+67val_67
+384val_384
+379val_379
+18val_18
+462val_462
+492val_492
+100val_100
+298val_298
+9val_9
+341val_341
+498val_498
+146val_146
+458val_458
+362val_362
+186val_186
+285val_285
+348val_348
+167val_167
+18val_18
+273val_273
+183val_183
+281val_281
+344val_344
+97val_97
+469val_469
+315val_315
+84val_84
+28val_28
+37val_37
+448val_448
+152val_152
+348val_348
+307val_307
+194val_194
+414val_414
+477val_477
+222val_222
+126val_126
+90val_90
+169val_169
+403val_403
+400val_400
+200val_200
+97val_97

src/contrib/hive/data/files/kv1_cc.txt

@@ -0,0 +1,500 @@
+238val_238
+86val_86
+311val_311
+27val_27
+165val_165
+409val_409
+255val_255
+278val_278
+98val_98
+484val_484
+265val_265
+193val_193
+401val_401
+150val_150
+273val_273
+224val_224
+369val_369
+66val_66
+128val_128
+213val_213
+146val_146
+406val_406
+429val_429
+374val_374
+152val_152
+469val_469
+145val_145
+495val_495
+37val_37
+327val_327
+281val_281
+277val_277
+209val_209
+15val_15
+82val_82
+403val_403
+166val_166
+417val_417
+430val_430
+252val_252
+292val_292
+219val_219
+287val_287
+153val_153
+193val_193
+338val_338
+446val_446
+459val_459
+394val_394
+237val_237
+482val_482
+174val_174
+413val_413
+494val_494
+207val_207
+199val_199
+466val_466
+208val_208
+174val_174
+399val_399
+396val_396
+247val_247
+417val_417
+489val_489
+162val_162
+377val_377
+397val_397
+309val_309
+365val_365
+266val_266
+439val_439
+342val_342
+367val_367
+325val_325
+167val_167
+195val_195
+475val_475
+17val_17
+113val_113
+155val_155
+203val_203
+339val_339
+0val_0
+455val_455
+128val_128
+311val_311
+316val_316
+57val_57
+302val_302
+205val_205
+149val_149
+438val_438
+345val_345
+129val_129
+170val_170
+20val_20
+489val_489
+157val_157
+378val_378
+221val_221
+92val_92
+111val_111
+47val_47
+72val_72
+4val_4
+280val_280
+35val_35
+427val_427
+277val_277
+208val_208
+356val_356
+399val_399
+169val_169
+382val_382
+498val_498
+125val_125
+386val_386
+437val_437
+469val_469
+192val_192
+286val_286
+187val_187
+176val_176
+54val_54
+459val_459
+51val_51
+138val_138
+103val_103
+239val_239
+213val_213
+216val_216
+430val_430
+278val_278
+176val_176
+289val_289
+221val_221
+65val_65
+318val_318
+332val_332
+311val_311
+275val_275
+137val_137
+241val_241
+83val_83
+333val_333
+180val_180
+284val_284
+12val_12
+230val_230
+181val_181
+67val_67
+260val_260
+404val_404
+384val_384
+489val_489
+353val_353
+373val_373
+272val_272
+138val_138
+217val_217
+84val_84
+348val_348
+466val_466
+58val_58
+8val_8
+411val_411
+230val_230
+208val_208
+348val_348
+24val_24
+463val_463
+431val_431
+179val_179
+172val_172
+42val_42
+129val_129
+158val_158
+119val_119
+496val_496
+0val_0
+322val_322
+197val_197
+468val_468
+393val_393
+454val_454
+100val_100
+298val_298
+199val_199
+191val_191
+418val_418
+96val_96
+26val_26
+165val_165
+327val_327
+230val_230
+205val_205
+120val_120
+131val_131
+51val_51
+404val_404
+43val_43
+436val_436
+156val_156
+469val_469
+468val_468
+308val_308
+95val_95
+196val_196
+288val_288
+481val_481
+457val_457
+98val_98
+282val_282
+197val_197
+187val_187
+318val_318
+318val_318
+409val_409
+470val_470
+137val_137
+369val_369
+316val_316
+169val_169
+413val_413
+85val_85
+77val_77
+0val_0
+490val_490
+87val_87
+364val_364
+179val_179
+118val_118
+134val_134
+395val_395
+282val_282
+138val_138
+238val_238
+419val_419
+15val_15
+118val_118
+72val_72
+90val_90
+307val_307
+19val_19
+435val_435
+10val_10
+277val_277
+273val_273
+306val_306
+224val_224
+309val_309
+389val_389
+327val_327
+242val_242
+369val_369
+392val_392
+272val_272
+331val_331
+401val_401
+242val_242
+452val_452
+177val_177
+226val_226
+5val_5
+497val_497
+402val_402
+396val_396
+317val_317
+395val_395
+58val_58
+35val_35
+336val_336
+95val_95
+11val_11
+168val_168
+34val_34
+229val_229
+233val_233
+143val_143
+472val_472
+322val_322
+498val_498
+160val_160
+195val_195
+42val_42
+321val_321
+430val_430
+119val_119
+489val_489
+458val_458
+78val_78
+76val_76
+41val_41
+223val_223
+492val_492
+149val_149
+449val_449
+218val_218
+228val_228
+138val_138
+453val_453
+30val_30
+209val_209
+64val_64
+468val_468
+76val_76
+74val_74
+342val_342
+69val_69
+230val_230
+33val_33
+368val_368
+103val_103
+296val_296
+113val_113
+216val_216
+367val_367
+344val_344
+167val_167
+274val_274
+219val_219
+239val_239
+485val_485
+116val_116
+223val_223
+256val_256
+263val_263
+70val_70
+487val_487
+480val_480
+401val_401
+288val_288
+191val_191
+5val_5
+244val_244
+438val_438
+128val_128
+467val_467
+432val_432
+202val_202
+316val_316
+229val_229
+469val_469
+463val_463
+280val_280
+2val_2
+35val_35
+283val_283
+331val_331
+235val_235
+80val_80
+44val_44
+193val_193
+321val_321
+335val_335
+104val_104
+466val_466
+366val_366
+175val_175
+403val_403
+483val_483
+53val_53
+105val_105
+257val_257
+406val_406
+409val_409
+190val_190
+406val_406
+401val_401
+114val_114
+258val_258
+90val_90
+203val_203
+262val_262
+348val_348
+424val_424
+12val_12
+396val_396
+201val_201
+217val_217
+164val_164
+431val_431
+454val_454
+478val_478
+298val_298
+125val_125
+431val_431
+164val_164
+424val_424
+187val_187
+382val_382
+5val_5
+70val_70
+397val_397
+480val_480
+291val_291
+24val_24
+351val_351
+255val_255
+104val_104
+70val_70
+163val_163
+438val_438
+119val_119
+414val_414
+200val_200
+491val_491
+237val_237
+439val_439
+360val_360
+248val_248
+479val_479
+305val_305
+417val_417
+199val_199
+444val_444
+120val_120
+429val_429
+169val_169
+443val_443
+323val_323
+325val_325
+277val_277
+230val_230
+478val_478
+178val_178
+468val_468
+310val_310
+317val_317
+333val_333
+493val_493
+460val_460
+207val_207
+249val_249
+265val_265
+480val_480
+83val_83
+136val_136
+353val_353
+172val_172
+214val_214
+462val_462
+233val_233
+406val_406
+133val_133
+175val_175
+189val_189
+454val_454
+375val_375
+401val_401
+421val_421
+407val_407
+384val_384
+256val_256
+26val_26
+134val_134
+67val_67
+384val_384
+379val_379
+18val_18
+462val_462
+492val_492
+100val_100
+298val_298
+9val_9
+341val_341
+498val_498
+146val_146
+458val_458
+362val_362
+186val_186
+285val_285
+348val_348
+167val_167
+18val_18
+273val_273
+183val_183
+281val_281
+344val_344
+97val_97
+469val_469
+315val_315
+84val_84
+28val_28
+37val_37
+448val_448
+152val_152
+348val_348
+307val_307
+194val_194
+414val_414
+477val_477
+222val_222
+126val_126
+90val_90
+169val_169
+403val_403
+400val_400
+200val_200
+97val_97

src/contrib/hive/data/files/kv4.txt

@@ -0,0 +1 @@
+邵铮

src/contrib/hive/metastore/if/hive_metastore.thrift

@@ -161,8 +161,9 @@ service ThriftHiveMetastore extends fb303.FacebookService
   // delete data (including partitions) if deleteData is set to true
   void drop_table(1:string dbname, 2:string name, 3:bool deleteData) 
                        throws(1:NoSuchObjectException o1, 2:MetaException o3)
-  list<string> get_tables(string db_name, string pattern)
-                       throws (MetaException o1, UnknownTableException o2, UnknownDBException o3)
+  list<string> get_tables(string db_name, string pattern) 
+                       throws (MetaException o1)
+
   Table get_table(1:string dbname, 2:string tbl_name) 
                        throws (1:MetaException o1, 2:NoSuchObjectException o2)
   // alter table applies to only future partitions not for existing partitions

src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java

@@ -77,6 +77,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
       };
 
       // The next serial number to be assigned
+      private boolean checkForDefaultDb;
       private static int nextSerialNum = 0;
       private static ThreadLocal<Integer> threadLocalId = new ThreadLocal() {
         protected synchronized Object initialValue() {
@@ -109,6 +110,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
       
       private boolean init() throws MetaException {
         rawStoreClassName = hiveConf.get("hive.metastore.rawstore.impl");
+        checkForDefaultDb = hiveConf.getBoolean("hive.metastore.checkForDefaultDb", true);
         wh = new Warehouse(hiveConf);
         createDefaultDB();
         return true;
@@ -134,7 +136,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
        * @throws MetaException
        */
       private void createDefaultDB() throws MetaException {
-        if(HMSHandler.createDefaultDB) {
+        if(HMSHandler.createDefaultDB || !checkForDefaultDb) {
           return;
         }
         try {
@@ -358,6 +360,11 @@ public class HiveMetaStore extends ThriftHiveMetastore {
           throws InvalidObjectException, AlreadyExistsException, MetaException {
         this.incrementCounter("append_partition");
         logStartFunction("append_partition", dbName, tableName);
+        if(LOG.isDebugEnabled()) {
+          for (String part : part_vals) {
+            LOG.debug(part);
+          }
+        }
         Partition part = new Partition();
         boolean success = false;
         try {
@@ -367,19 +374,20 @@ public class HiveMetaStore extends ThriftHiveMetastore {
           part.setTableName(tableName);
           part.setValues(part_vals);
 
-          Partition old_part = this.get_partition(part.getDbName(), part.getTableName(), part.getValues());
-          if( old_part != null) {
-            throw new AlreadyExistsException("Partition already exists:" + part);
-          }
-          
           Table tbl = getMS().getTable(part.getDbName(), part.getTableName());
           if(tbl == null) {
             throw new InvalidObjectException("Unable to add partition because table or database do not exist");
           }
+
           part.setSd(tbl.getSd());
           Path partLocation = new Path(tbl.getSd().getLocation(), Warehouse.makePartName(tbl.getPartitionKeys(), part_vals));
           part.getSd().setLocation(partLocation.toString());
 
+          Partition old_part = this.get_partition(part.getDbName(), part.getTableName(), part.getValues());
+          if( old_part != null) {
+            throw new AlreadyExistsException("Partition already exists:" + part);
+          }
+          
           success = getMS().addPartition(part);
           if(success) {
             success = getMS().commitTransaction();
@@ -454,8 +462,32 @@ public class HiveMetaStore extends ThriftHiveMetastore {
           TException {
         this.incrementCounter("drop_partition");
         logStartFunction("drop_partition", db_name, tbl_name);
-        // TODO:pc drop the data as needed
-        return getMS().dropPartition(db_name, tbl_name, part_vals);
+        LOG.info("Partition values:" + part_vals);
+        boolean success = false;
+        Path partPath = null;
+        try {
+          getMS().openTransaction();
+          Partition part = this.get_partition(db_name, tbl_name, part_vals);
+          if(part == null) {
+            throw new NoSuchObjectException("Partition doesn't exist. " + part_vals);
+          }
+          if(part.getSd() == null  || part.getSd().getLocation() == null) {
+            throw new MetaException("Partition metadata is corrupted");
+          }
+          if(!getMS().dropPartition(db_name, tbl_name, part_vals)) {
+            throw new MetaException("Unable to drop partition");
+          }
+          success  = getMS().commitTransaction();
+          partPath = new Path(part.getSd().getLocation());
+        } finally {
+          if(!success) {
+            getMS().rollbackTransaction();
+          } else if(deleteData && (partPath != null)) {
+            wh.deleteDir(partPath, true);
+            // ok even if the data is not deleted
+          }
+        }
+        return true;
       }
 
       public Partition get_partition(String db_name, String tbl_name, List<String> part_vals)

src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java

@@ -142,12 +142,13 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
     if(!open) {
       throw new MetaException("Could not connect to meta store using any of the URIs provided");
     }
+    LOG.info("Connected to metastore.");
   }
  
   private void openStore(URI store) throws MetaException {
     open = false;
     transport = new TSocket(store.getHost(), store.getPort());
-    ((TSocket)transport).setTimeout(2000);
+    ((TSocket)transport).setTimeout(20000);
     TProtocol protocol = new TBinaryProtocol(transport);
     client = new ThriftHiveMetastore.Client(protocol);
 

src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java

@@ -108,4 +108,18 @@ public interface IMetaStoreClient {
   public void alter_table(String defaultDatabaseName, String tblName, Table table) throws InvalidOperationException, MetaException, TException;
   public boolean createDatabase(String name, String location_uri) throws AlreadyExistsException, MetaException, TException;
   public boolean dropDatabase(String name) throws MetaException, TException;
+
+  /**
+   * @param db_name
+   * @param tbl_name
+   * @param part_vals
+   * @param deleteData delete the underlying data or just delete the table in metadata
+   * @return
+   * @throws NoSuchObjectException
+   * @throws MetaException
+   * @throws TException
+   * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_partition(java.lang.String, java.lang.String, java.util.List)
+   */
+  public boolean dropPartition(String db_name, String tbl_name, List<String> part_vals, boolean deleteData)
+      throws NoSuchObjectException, MetaException, TException;
 }

src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreClient.java

@@ -439,4 +439,9 @@ public class MetaStoreClient implements IMetaStoreClient {
     return new ArrayList<String>();
   }
 
+  public boolean dropPartition(String db_name, String tbl_name, List<String> part_vals, boolean deleteData)
+      throws NoSuchObjectException, MetaException, TException {
+    return false;
+  }
+
 }

src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java

@@ -102,7 +102,7 @@ public class MetaStoreUtils {
       tTable.getPartitionKeys().add(part);
     }
     // not sure why these are needed
-    serdeInfo.setSerializationLib(MetadataTypedColumnsetSerDe.shortName());
+    serdeInfo.setSerializationLib(MetadataTypedColumnsetSerDe.class.getName());
     sd.setNumBuckets(-1);
     return tTable;
   }
@@ -266,14 +266,22 @@ public class MetaStoreUtils {
       oldName = oldName.replace("com.facebook.thrift.hive.MetadataTypedColumnsetSerDe",org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
 
       // columnset serde
-      oldName = oldName.replace("com.facebook.thrift.hive.columnsetSerDe",org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
+      oldName = oldName.replace("com.facebook.thrift.hive.columnsetSerDe",org.apache.hadoop.hive.serde.thrift.columnsetSerDe.class.getName());
       oldName = oldName.replace("org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe",
-          org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
-
+      	  org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
+      oldName = oldName.replace("com.facebook.thrift.hive.MetadataTypedColumnsetSerDe", org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
       // thrift serde
-      oldName = oldName.replace("com.facebook.thrift.hive.ThriftHiveSerDe",org.apache.hadoop.hive.serde2.ThriftDeserializer.class.getName());
-      oldName = oldName.replace("org.apache.hadoop.hive.serde.thrift.ThriftSerDe",
-          org.apache.hadoop.hive.serde2.ThriftDeserializer.class.getName());
+      oldName = oldName.replace("com.facebook.thrift.hive.ThriftHiveSerDe", org.apache.hadoop.hive.serde2.ThriftDeserializer.class.getName());
+      oldName = oldName.replace("org.apache.hadoop.hive.serde.thrift.ThriftSerDe", org.apache.hadoop.hive.serde2.ThriftDeserializer.class.getName());
+
+      // replace any old short names in filebased metadata
+      if(oldName.equals("columnset"))
+        oldName = org.apache.hadoop.hive.serde.thrift.columnsetSerDe.class.getName();
+      if(oldName.equals("simple_meta"))
+        oldName = org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName();
+      if(oldName.equals("thrift"))
+        oldName = org.apache.hadoop.hive.serde2.ThriftDeserializer.class.getName();
+
       p.setProperty(key,oldName);
     }
     return p;
@@ -328,6 +336,8 @@ public class MetaStoreUtils {
     }
     // needed for MetadataTypedColumnSetSerDe
     setSerdeParam(t.getSd().getSerdeInfo(), schema, org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_COLUMNS);
+    // needed for DynamicSerDe
+    setSerdeParam(t.getSd().getSerdeInfo(), schema, org.apache.hadoop.hive.serde.Constants.SERIALIZATION_DDL);
     
     String colstr = schema.getProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_COLUMNS);
     List<FieldSchema>  fields = new ArrayList<FieldSchema>();
@@ -350,6 +360,8 @@ public class MetaStoreUtils {
     }
     t.getSd().setCols(fields);
     
+    t.setOwner(schema.getProperty("creator"));
+    
     // remove all the used up parameters to find out the remaining parameters
     schema.remove(Constants.META_TABLE_NAME);
     schema.remove(Constants.META_TABLE_LOCATION);
@@ -373,13 +385,70 @@ public class MetaStoreUtils {
     return t;
   }
 
-  private static void setSerdeParam(SerDeInfo sdi, Properties schema, String param) {
+  public static void setSerdeParam(SerDeInfo sdi, Properties schema, String param) {
     String val = schema.getProperty(param);
     if(org.apache.commons.lang.StringUtils.isNotBlank(val)) {
       sdi.getParameters().put(param, val);
     }
   }
 
+  static HashMap<String, String> typeToThriftTypeMap; 
+  static {
+    typeToThriftTypeMap = new HashMap<String, String>();
+    typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.TINYINT_TYPE_NAME, "byte");
+    typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.INT_TYPE_NAME, "i32");
+    typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.BIGINT_TYPE_NAME, "i64");
+    typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.DOUBLE_TYPE_NAME, "double");
+    typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.FLOAT_TYPE_NAME, "float");
+    typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.LIST_TYPE_NAME, "list");
+    typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.MAP_TYPE_NAME, "map");
+    typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.STRING_TYPE_NAME, "string");
+    // These 3 types are not supported yet. 
+    // We should define a complex type date in thrift that contains a single int member, and DynamicSerDe
+    // should convert it to date type at runtime.
+    typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.DATE_TYPE_NAME, "date");
+    typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.DATETIME_TYPE_NAME, "datetime");
+    typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.TIMESTAMP_TYPE_NAME, "timestamp");
+  }
+  /** Convert type to ThriftType.  We do that by tokenizing the type and convert each token.
+   */
+  public static String typeToThriftType(String type) {
+    StringBuilder thriftType = new StringBuilder();
+    int last = 0;
+    boolean lastAlphaDigit = Character.isLetterOrDigit(type.charAt(last));
+    for(int i=1; i<=type.length(); i++) {
+      if (i == type.length() || Character.isLetterOrDigit(type.charAt(i)) != lastAlphaDigit) {
+        String token = type.substring(last, i);
+        last = i;
+        String thriftToken = typeToThriftTypeMap.get(token);
+        thriftType.append(thriftToken == null? token : thriftToken);
+        lastAlphaDigit = !lastAlphaDigit;
+      }         
+    }
+    return thriftType.toString();
+  }
+  /** Convert FieldSchemas to Thrift DDL.
+   */
+  public static String getDDLFromFieldSchema(String structName, List<FieldSchema> fieldSchemas) {
+    StringBuilder ddl = new StringBuilder();
+    ddl.append("struct ");
+    ddl.append(structName);
+    ddl.append(" { ");
+    boolean first = true;
+    for (FieldSchema col: fieldSchemas) {
+      if (first) {
+        first = false;
+      } else {
+        ddl.append(", ");
+      }
+      ddl.append(typeToThriftType(col.getType()));
+      ddl.append(' ');
+      ddl.append(col.getName());
+    }
+    ddl.append("}");
+    LOG.warn("DDL: " + ddl);
+    return ddl.toString();
+  }
   public static Properties getSchema(org.apache.hadoop.hive.metastore.api.Table tbl) {
     Properties schema = new Properties();
     String inputFormat = tbl.getSd().getInputFormat();
@@ -404,7 +473,7 @@ public class MetaStoreUtils {
     if(tbl.getSd().getSerdeInfo().getSerializationLib() != null) {
       schema.setProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_LIB, tbl.getSd().getSerdeInfo().getSerializationLib());
     }
-    StringBuffer buf = new StringBuffer();
+    StringBuilder buf = new StringBuilder();
     boolean first = true;
     for (FieldSchema col: tbl.getSd().getCols()) {
       if (!first) {
@@ -415,6 +484,8 @@ public class MetaStoreUtils {
     }
     String cols = buf.toString();
     schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_COLUMNS, cols);
+    schema.setProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_DDL, 
+        getDDLFromFieldSchema(tbl.getTableName(), tbl.getSd().getCols()));
     
     String partString = "";
     String partStringSep = "";
@@ -429,10 +500,6 @@ public class MetaStoreUtils {
       schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_PARTITION_COLUMNS, partString);
     }
     
-    //TODO:pc field_to_dimension doesn't seem to be used anywhere so skipping for now
-    schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.BUCKET_FIELD_NAME, "");
-    schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.FIELD_TO_DIMENSION, "");
-    
     for(Entry<String, String> e: tbl.getParameters().entrySet()) {
       schema.setProperty(e.getKey(), e.getValue());
     }

src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java

@@ -528,11 +528,24 @@ public class ObjectStore implements RawStore, Configurable {
     try {
       openTransaction();
       dbName = dbName.toLowerCase();
-      pattern = "(?i)" + pattern; // add the case insensitivity 
-      Query q = pm.newQuery("select tableName from org.apache.hadoop.hive.metastore.model.MTable where database.name == dbName && tableName.matches(pattern)");
-      q.declareParameters("java.lang.String dbName, java.lang.String pattern");
+      // Take the pattern and split it on the | to get all the composing patterns
+      String [] subpatterns = pattern.trim().split("\\|");
+      String query = "select tableName from org.apache.hadoop.hive.metastore.model.MTable where database.name == dbName && (";
+      boolean first = true;
+      for(String subpattern: subpatterns) {
+        subpattern = "(?i)" + subpattern.replaceAll("\\*", ".*");
+        if (!first) {
+          query = query + " || ";
+        }
+        query = query + " tableName.matches(\"" + subpattern + "\")";
+        first = false;
+      }
+      query = query + ")";
+
+      Query q = pm.newQuery(query);
+      q.declareParameters("java.lang.String dbName");
       q.setResult("tableName");
-      Collection names = (Collection) q.execute(dbName.trim(), pattern.trim());
+      Collection names = (Collection) q.execute(dbName.trim());
       tbls = new ArrayList<String>(); 
       for (Iterator i = names.iterator (); i.hasNext ();) {
           tbls.add((String) i.next ()); 
@@ -817,7 +830,7 @@ public class ObjectStore implements RawStore, Configurable {
       LOG.debug("Executing getPartitionNames");
       dbName = dbName.toLowerCase();
       tableName = tableName.toLowerCase();
-      Query q = pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition where table.database.name == t1 && table.tableName == t2");
+      Query q = pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition where table.database.name == t1 && table.tableName == t2 order by partitionName asc");
       q.declareParameters("java.lang.String t1, java.lang.String t2");
       q.setResult("partitionName");
       Collection names = (Collection) q.execute(dbName.trim(), tableName.trim());
@@ -847,9 +860,10 @@ public class ObjectStore implements RawStore, Configurable {
       Query query = pm.newQuery(MPartition.class, "table.tableName == t1 && table.database.name == t2"); 
       query.declareParameters("java.lang.String t1, java.lang.String t2"); 
       mparts = (List<MPartition>) query.execute(tableName.trim(), dbName.trim()); 
+      LOG.debug("Done executing query for listMPartitions");
       pm.retrieveAll(mparts);
       success = commitTransaction();
-      LOG.debug("Done e xecuting listMPartitions");
+      LOG.debug("Done retrieving all objects for listMPartitions");
     } finally {
       if(!success) {
         rollbackTransaction();

src/contrib/hive/ql/build.xml

@@ -64,6 +64,13 @@
               resultsDirectory="${ql.test.results.dir}/clientpositive" className="TestCliDriver"
               logFile="${test.log.dir}/testclidrivergen.log"/>
 
+    <qtestgen outputDirectory="${test.build.src}/org/apache/hadoop/hive/cli" 
+              templatePath="${ql.test.template.dir}" template="TestNegativeCliDriver.vm" 
+              queryDirectory="${ql.test.query.dir}/clientnegative" 
+              queryFile="${qfile}"
+              resultsDirectory="${ql.test.results.dir}/clientnegative" className="TestNegativeCliDriver"
+              logFile="${test.log.dir}/testnegclidrivergen.log"/>
+
   </target>
 
   <uptodate property="grammarBuild.notRequired">

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Context.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql;
 
+import java.io.File;
 import java.io.DataInput;
 import java.io.IOException;
 import java.io.FileNotFoundException;
@@ -29,6 +30,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.util.StringUtils;
+import java.util.Random;
 
 public class Context {
   private Path resFile;
@@ -38,9 +40,12 @@ public class Context {
   private Path[] resDirPaths;
   private int    resDirFilesNum;
   boolean initialized;
+  private String scratchDir;
+  private HiveConf conf;
   
   public Context(HiveConf conf) {
     try {
+      this.conf = conf;
       fs = FileSystem.get(conf);
       initialized = false;
       resDir = null;
@@ -50,6 +55,23 @@ public class Context {
     }
   }
 
+  public void makeScratchDir() throws Exception {
+    Random rand = new Random();
+    int randomid = Math.abs(rand.nextInt()%rand.nextInt());
+    scratchDir = conf.getVar(HiveConf.ConfVars.SCRATCHDIR) + File.separator + randomid;
+    Path tmpdir = new Path(scratchDir);
+    fs.mkdirs(tmpdir);
+  }
+
+  public String getScratchDir() {
+    return scratchDir;
+  }
+
+  public void removeScratchDir() throws Exception {
+    Path tmpdir = new Path(scratchDir);
+    fs.delete(tmpdir, true);
+  }
+
   /**
    * @return the resFile
    */

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Driver.java

@@ -34,10 +34,8 @@ import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactory;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
 import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.exec.MapRedTask;
 import org.apache.hadoop.hive.ql.exec.TaskFactory;
 import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.ExecDriver;
 import org.apache.hadoop.hive.serde.ByteStream;
 import org.apache.hadoop.hive.conf.HiveConf;
 
@@ -55,13 +53,14 @@ public class Driver implements CommandProcessor {
   private DataInput    resStream;
   private LogHelper    console;
   private Context      ctx;
+  private BaseSemanticAnalyzer sem;
   
   public int countJobs(List<Task<? extends Serializable>> tasks) {
     if (tasks == null)
       return 0;
     int jobs = 0;
     for (Task<? extends Serializable> task: tasks) {
-      if ((task instanceof ExecDriver) || (task instanceof MapRedTask)) {
+      if (task.isMapRedTask()) {
         jobs++;
       }
       jobs += countJobs(task.getChildTasks());
@@ -69,6 +68,22 @@ public class Driver implements CommandProcessor {
     return jobs;
   }
 
+  public boolean hasReduceTasks(List<Task<? extends Serializable>> tasks) {
+    if (tasks == null)
+      return false;
+
+    boolean hasReduce = false;
+    for (Task<? extends Serializable> task: tasks) {
+      if (task.hasReduce()) {
+        return true;
+      }
+
+      hasReduce = (hasReduce || hasReduceTasks(task.getChildTasks()));
+    }
+    return hasReduce;
+  }
+
+
   /**
    * for backwards compatibility with current tests
    */ 
@@ -97,11 +112,10 @@ public class Driver implements CommandProcessor {
     try {
       
       TaskFactory.resetId();
-
-      BaseSemanticAnalyzer sem;
       LOG.info("Starting command: " + command);
 
       ctx.clear();
+      ctx.makeScratchDir();
       resStream = null;
       
       pd = new ParseDriver();
@@ -122,12 +136,18 @@ public class Driver implements CommandProcessor {
         console.printInfo("Total MapReduce jobs = " + jobs);
       }
       
- 
+      boolean hasReduce = hasReduceTasks(sem.getRootTasks());
+      if (hasReduce) {
+        console.printInfo("Number of reducers = " + conf.getIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS));
+        console.printInfo("In order to change numer of reducers use:");
+        console.printInfo("  set mapred.reduce.tasks = <number>");
+      }
+
       String jobname = Utilities.abbreviate(command, maxlen - 6);
       int curJob = 0;
       for(Task<? extends Serializable> rootTask: sem.getRootTasks()) {
         // assumption that only top level tasks are map-reduce tasks
-        if ((rootTask instanceof ExecDriver) || (rootTask instanceof MapRedTask)) {
+        if (rootTask.isMapRedTask()) {
           curJob ++;
           if(noName) {
             conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, jobname + "(" + curJob + "/" + jobs + ")");
@@ -175,10 +195,10 @@ public class Driver implements CommandProcessor {
         }
       }
     } catch (SemanticException e) {
-      console.printError("FAILED: Error in semantic analysis: " + e.getMessage());
+      console.printError("FAILED: Error in semantic analysis: " + e.getMessage(), "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
       return (10);
     } catch (ParseException e) {
-      console.printError("FAILED: Parse Error: " + e.getMessage());
+      console.printError("FAILED: Parse Error: " + e.getMessage(), "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
       return (11);
     } catch (Exception e) {
       // Has to use full name to make sure it does not conflict with org.apache.commons.lang.StringUtils
@@ -196,14 +216,23 @@ public class Driver implements CommandProcessor {
   }
   
   
-  public boolean getResults(Vector<Vector<String>> res) 
+  public boolean getResults(Vector<String> res) 
   {
+  	if (sem.getFetchTask() != null) {
+      if (!sem.getFetchTaskInit()) {
+        sem.setFetchTaskInit(true);
+        sem.getFetchTask().initialize(conf);
+      }
+  		boolean ret = sem.getFetchTask().fetch(res);
+  		return ret;  		
+  	}
+
     if (resStream == null)
       resStream = ctx.getStream();
     if (resStream == null) return false;
     
     int numRows = 0;
-    Vector<String> row = new Vector<String>();
+    String row = null;
 
     while (numRows < MAX_ROWS)
     {
@@ -215,47 +244,45 @@ public class Driver implements CommandProcessor {
           return false;
       }
 
-      String col = null;
       bos.reset();
-      Utilities.streamStatus ss = Utilities.streamStatus.NORMAL;
+      Utilities.streamStatus ss;
       try
       {
         ss = Utilities.readColumn(resStream, bos);
         if (bos.getCount() > 0)
-          col = new String(bos.getData(), 0, bos.getCount(), "UTF-8");
-        else if (ss == Utilities.streamStatus.NORMAL)
-          col = Utilities.NSTR;
+          row = new String(bos.getData(), 0, bos.getCount(), "UTF-8");
+        else if (ss == Utilities.streamStatus.TERMINATED)
+          row = new String();
+
+        if (row != null) {
+          numRows++;
+          res.add(row);
+        }
       } catch (IOException e) {
         console.printError("FAILED: Unexpected IO exception : " + e.getMessage());
         res = null;
         return false;
       }
-      
-      if ((ss == Utilities.streamStatus.EOF) || 
-          (ss == Utilities.streamStatus.TERMINATED))
-      {
-        if (col != null) 
-          row.add(col.equals(Utilities.nullStringStorage) ? null : col);
-        else if (row.size() != 0) 
-          row.add(null);
-
-        numRows++;
-        res.add(row);
-        row = new Vector<String>();
-        col = null;
-
-        if (ss == Utilities.streamStatus.EOF) 
-          resStream = ctx.getStream();
-      }
-      else if (ss == Utilities.streamStatus.NORMAL)
-      {
-        row.add(col.equals(Utilities.nullStringStorage) ? null : col);
-        col = null;
-      }
-      else
-        assert false;
+
+      if (ss == Utilities.streamStatus.EOF) 
+        resStream = ctx.getStream();
     }
     return true;
   }
+
+  public int close() {
+    try {
+      // Delete the scratch directory from the context
+      ctx.removeScratchDir();
+      ctx.clear();
+    }
+    catch (Exception e) {
+      console.printError("FAILED: Unknown exception : " + e.getMessage(),
+                         "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
+      return(13);
+    }
+    
+    return(0);
+  }
 }
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java

@@ -48,16 +48,14 @@ public class ColumnInfo implements Serializable {
   public ColumnInfo() {
   }
 
-  public ColumnInfo(String internalName, TypeInfo type, boolean isVirtual) {
+  public ColumnInfo(String internalName, TypeInfo type) {
     this.internalName = internalName;
     this.type = type;
-    this.isVirtual = isVirtual;
   }
   
-  public ColumnInfo(String internalName, Class type, boolean isVirtual) {
+  public ColumnInfo(String internalName, Class type) {
     this.internalName = internalName;
     this.type = TypeInfoFactory.getPrimitiveTypeInfo(type);
-    this.isVirtual = isVirtual;
   }
   
   public TypeInfo getType() {
@@ -67,10 +65,6 @@ public class ColumnInfo implements Serializable {
   public String getInternalName() {
     return internalName;
   }
-
-  public boolean getIsVirtual() {
-    return isVirtual;
-  }
   
   public void setType(TypeInfo type) {
     this.type = type;
@@ -79,9 +73,4 @@ public class ColumnInfo implements Serializable {
   public void setInternalName(String internalName) {
     this.internalName = internalName;
   }
-
-  public void setIsVirtual(boolean isVirtual) {
-    this.isVirtual = isVirtual;
-  }
-
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java

@@ -22,39 +22,46 @@ import java.io.DataOutput;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.SortedSet;
 import java.util.TreeSet;
 
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Order;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.plan.DDLWork;
 import org.apache.hadoop.hive.ql.plan.alterTableDesc;
 import org.apache.hadoop.hive.ql.plan.createTableDesc;
 import org.apache.hadoop.hive.ql.plan.descTableDesc;
 import org.apache.hadoop.hive.ql.plan.dropTableDesc;
+import org.apache.hadoop.hive.ql.plan.showPartitionsDesc;
 import org.apache.hadoop.hive.ql.plan.showTablesDesc;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde.thrift.columnsetSerDe;
+import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.mapred.TextInputFormat;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.serde.Constants;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+
 import com.facebook.thrift.TException;
 
 /**
@@ -66,7 +73,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
   static final private Log LOG = LogFactory.getLog("hive.ql.exec.DDLTask");
 
   transient HiveConf conf;
-  static final private int separator  = Utilities.ctrlaCode;
+  static final private int separator  = Utilities.tabCode;
   static final private int terminator = Utilities.newLineCode;
   
   public void initialize(HiveConf conf) {
@@ -98,14 +105,45 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
           tbl.setPartCols(crtTbl.getPartCols());
         if (crtTbl.getNumBuckets() != -1)
           tblStorDesc.setNumBuckets(crtTbl.getNumBuckets());
-        if (crtTbl.getFieldDelim() != null)
-          tbl.setSerdeParam(Constants.FIELD_DELIM, crtTbl.getFieldDelim());
-        if (crtTbl.getCollItemDelim() != null)
-          tbl.setSerdeParam(Constants.COLLECTION_DELIM, crtTbl.getCollItemDelim());
-        if (crtTbl.getMapKeyDelim() != null)
-          tbl.setSerdeParam(Constants.MAPKEY_DELIM, crtTbl.getMapKeyDelim());
-        if (crtTbl.getLineDelim() != null)
-          tbl.setSerdeParam(Constants.LINE_DELIM, crtTbl.getLineDelim());
+
+        if (crtTbl.getSerName() != null) {
+        	tbl.setSerializationLib(crtTbl.getSerName());
+          if (crtTbl.getMapProp() != null) {
+            Iterator<Map.Entry<String, String>> iter = crtTbl.getMapProp().entrySet().iterator();
+            while (iter.hasNext()) {
+              Map.Entry<String, String> m = (Map.Entry)iter.next();
+              tbl.setSerdeParam(m.getKey(), m.getValue());
+            }
+          }
+        } 
+        else
+        {
+          if (crtTbl.getFieldDelim() != null)
+          {
+            tbl.setSerdeParam(Constants.FIELD_DELIM, crtTbl.getFieldDelim());
+            tbl.setSerdeParam(Constants.SERIALIZATION_FORMAT, crtTbl.getFieldDelim());
+          }
+        
+          if (crtTbl.getCollItemDelim() != null)
+            tbl.setSerdeParam(Constants.COLLECTION_DELIM, crtTbl.getCollItemDelim());
+          if (crtTbl.getMapKeyDelim() != null)
+            tbl.setSerdeParam(Constants.MAPKEY_DELIM, crtTbl.getMapKeyDelim());
+          if (crtTbl.getLineDelim() != null)
+            tbl.setSerdeParam(Constants.LINE_DELIM, crtTbl.getLineDelim());
+        }
+        
+        /**
+         * For now, if the user specifies either the map or the collections delimiter, we infer the 
+         * table to DynamicSerDe/TCTLSeparatedProtocol.
+         * In the future, we should infer this for any delimiters specified, but this will break older
+         * hive tables, so not for now. 
+         */
+        if (crtTbl.getCollItemDelim() != null || crtTbl.getMapKeyDelim() != null) {
+          tbl.setSerializationLib(org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe.class.getName());
+          tbl.setSerdeParam(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName());
+        }
+
+
         if (crtTbl.getComment() != null)
           tbl.setProperty("comment", crtTbl.getComment());
         if (crtTbl.getLocation() != null)
@@ -157,6 +195,11 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
               tbl.setProperty("SORTBUCKETCOLSPREFIX", "TRUE");
           }
         }
+        
+        // set owner, create_time etc
+        tbl.setOwner(System.getProperty("user.name"));
+        // set create time
+        tbl.getTTable().setCreateTime((int) (System.currentTimeMillis()/1000));
 
         // create the table
         db.createTable(tbl);
@@ -165,8 +208,30 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
 
       dropTableDesc dropTbl = work.getDropTblDesc();
       if (dropTbl != null) {
-        // drop the table
-        db.dropTable(dropTbl.getTableName());
+        if(dropTbl.getPartSpecs() == null) {
+          // drop the table
+          db.dropTable(dropTbl.getTableName());
+        } else {
+          // drop partitions in the list
+          Table tbl  = db.getTable(dropTbl.getTableName());
+          List<Partition> parts = new ArrayList<Partition>();
+          for(HashMap<String, String> partSpec : dropTbl.getPartSpecs()) {
+            Partition part = db.getPartition(tbl, partSpec, false);
+            if(part == null) {
+              console.printInfo("Partition " + partSpec + " does not exist.");
+            } else {
+              parts.add(part);
+            }
+          }
+          // drop all existing partitions from the list
+          for (Partition partition : parts) {
+            console.printInfo("Dropping the partition " + partition.getName());
+            db.dropPartition(MetaStoreUtils.DEFAULT_DATABASE_NAME, 
+                dropTbl.getTableName(), 
+                partition.getValues(), 
+                true); //drop data for the partition
+          }
+        }
         return 0;
       }
 
@@ -174,10 +239,55 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
       if (alterTbl != null) {
         // alter the table
         Table tbl = db.getTable(alterTbl.getOldName());
-         if (alterTbl.getOp() == alterTableDesc.alterTableTypes.RENAME)
-           tbl.getTTable().setTableName(alterTbl.getNewName());
-         else
-           tbl.getTTable().getSd().setCols(alterTbl.getNewCols());
+        if (alterTbl.getOp() == alterTableDesc.alterTableTypes.RENAME)
+          tbl.getTTable().setTableName(alterTbl.getNewName());
+        else if(alterTbl.getOp() == alterTableDesc.alterTableTypes.ADDCOLS) {
+          List<FieldSchema> newCols = alterTbl.getNewCols();
+          List<FieldSchema> oldCols = tbl.getCols();
+          if(tbl.getSerializationLib().equals(columnsetSerDe.class.getName())) {
+            console.printInfo("Replacing columns for columnsetSerDe and changing to typed SerDe");
+            tbl.setSerializationLib(MetadataTypedColumnsetSerDe.class.getName());
+            tbl.getTTable().getSd().setCols(newCols);
+          } 
+          else { 
+            // make sure the columns does not already exist
+            Iterator<FieldSchema> iterNewCols = newCols.iterator();
+            while (iterNewCols.hasNext()) {
+              FieldSchema newCol = iterNewCols.next();
+              String newColName  = newCol.getName();
+              Iterator<FieldSchema> iterOldCols = oldCols.iterator();
+              while (iterOldCols.hasNext()) {
+                String oldColName = iterOldCols.next().getName();
+                if (oldColName.equalsIgnoreCase(newColName)) { 
+                  console.printError("Column '" + newColName + "' exists");
+                  return 1;
+                }
+              }
+              oldCols.add(newCol);
+            }
+            tbl.getTTable().getSd().setCols(oldCols);
+          }
+        } 
+        else if(alterTbl.getOp() == alterTableDesc.alterTableTypes.REPLACECOLS) {
+          // change SerDe to MetadataTypedColumnsetSerDe if it is columnsetSerDe
+          if(tbl.getSerializationLib().equals(columnsetSerDe.class.getName())) {
+            console.printInfo("Replacing columns for columnsetSerDe and changing to typed SerDe");
+            tbl.setSerializationLib(MetadataTypedColumnsetSerDe.class.getName());
+          }
+          else if(!tbl.getSerializationLib().equals(MetadataTypedColumnsetSerDe.class.getName())) {
+            console.printError("Replace columns is not supported for this table. SerDe may be incompatible.");
+            return 1;
+          }
+          tbl.getTTable().getSd().setCols(alterTbl.getNewCols());
+        }
+        else {
+          console.printError("Unsupported Alter commnad");
+          return 1;
+        }
+
+        // set last modified by properties
+        tbl.setProperty("last_modified_by", System.getProperty("user.name"));
+        tbl.setProperty("last_modified_time", Long.toString(System.currentTimeMillis()/1000));
 
         try {
           db.alterTable(alterTbl.getOldName(), tbl);
@@ -194,17 +304,46 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
 
       descTableDesc descTbl = work.getDescTblDesc();
       if (descTbl != null) {
-        boolean found = true;
-
+        // describe the table - populate the output stream
+        Table tbl = db.getTable(descTbl.getTableName(), false);
+        Partition part = null;
         try {
-          // describe the table - populate the output stream
-          Table tbl = db.getTable(descTbl.getTableName());
-          
+          if(tbl == null) {
+            DataOutput outStream = (DataOutput)fs.open(descTbl.getResFile());
+            String errMsg = "Table " + descTbl.getTableName() + " does not exist";
+            outStream.write(errMsg.getBytes("UTF-8"));
+            ((FSDataOutputStream)outStream).close();
+            return 0;
+          }
+          if(descTbl.getPartSpec() != null) {
+            part = db.getPartition(tbl, descTbl.getPartSpec(), false);
+            if(part == null) {
+              DataOutput outStream = (DataOutput)fs.open(descTbl.getResFile());
+              String errMsg = "Partition " + descTbl.getPartSpec() + " for table " + descTbl.getTableName() + " does not exist";
+              outStream.write(errMsg.getBytes("UTF-8"));
+              ((FSDataOutputStream)outStream).close();
+              return 0;
+            }
+          }
+        } catch (FileNotFoundException e) {
+          LOG.info("describe table: " + StringUtils.stringifyException(e));
+          return 1;
+        }
+        catch (IOException e) {
+          LOG.info("describe table: " + StringUtils.stringifyException(e));
+          return 1;
+        }
+        
+        try {
+
           LOG.info("DDLTask: got data for " +  tbl.getName());
           
           // write the results in the file
           DataOutput os = (DataOutput)fs.create(descTbl.getResFile());
           List<FieldSchema> cols = tbl.getCols();
+          if(part != null) {
+            cols = part.getTPartition().getSd().getCols();
+          }
           Iterator<FieldSchema> iterCols = cols.iterator();
           boolean firstCol = true;
           while (iterCols.hasNext())
@@ -239,6 +378,19 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
               os.write(col.getComment().getBytes("UTF-8"));
             }
           }
+          
+          // if extended desc table then show the complete details of the table
+          if(descTbl.isExt()) {
+            if(part != null) {
+              // show partition informatio
+              os.write("\n\nDetailed Partition Information:\n".getBytes("UTF-8"));
+              os.write(part.getTPartition().toString().getBytes("UTF-8"));
+            } else {
+              os.write("\nDetailed Table Information:\n".getBytes("UTF-8"));
+              os.write(tbl.getTTable().toString().getBytes("UTF-8"));
+            }
+          }
+          
           LOG.info("DDLTask: written data for " +  tbl.getName());
           ((FSDataOutputStream)os).close();
           
@@ -246,30 +398,10 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
           LOG.info("describe table: " + StringUtils.stringifyException(e));
           return 1;
         }
-        catch (InvalidTableException e) {
-          found = false;
-        }
         catch (IOException e) {
           LOG.info("describe table: " + StringUtils.stringifyException(e));
           return 1;
         }
-
-        if (!found)
-        {
-          try {
-            DataOutput outStream = (DataOutput)fs.open(descTbl.getResFile());
-            String errMsg = "Table " + descTbl.getTableName() + " does not exist";
-            outStream.write(errMsg.getBytes("UTF-8"));
-            ((FSDataOutputStream)outStream).close();
-          } catch (FileNotFoundException e) {
-            LOG.info("describe table: " + StringUtils.stringifyException(e));
-            return 1;
-          }
-          catch (IOException e) {
-            LOG.info("describe table: " + StringUtils.stringifyException(e));
-            return 1;
-          }
-        }
         return 0;
       }
 
@@ -310,7 +442,52 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
         return 0;
       }
 
-    } catch (HiveException e) {
+      showPartitionsDesc showParts = work.getShowPartsDesc();
+      if (showParts != null) {
+        // get the partitions for the table and populate the output
+        String tabName = showParts.getTabName();
+        Table tbl = null;
+        List<String> parts = null;
+
+        tbl = db.getTable(tabName);
+
+        if (!tbl.isPartitioned()) {
+          console.printError("Table " + tabName + " is not a partitioned table");
+          return 1;
+        }
+
+        parts = db.getPartitionNames(MetaStoreUtils.DEFAULT_DATABASE_NAME, tbl.getName(), Short.MAX_VALUE);
+
+        // write the results in the file
+        try {
+          DataOutput outStream = (DataOutput)fs.create(showParts.getResFile());
+          Iterator<String> iterParts = parts.iterator();
+          boolean firstCol = true;
+          while (iterParts.hasNext())
+          {
+            if (!firstCol)
+              outStream.write(terminator);
+            outStream.write(iterParts.next().getBytes("UTF-8"));
+            firstCol = false;
+          }
+          ((FSDataOutputStream)outStream).close();
+        } catch (FileNotFoundException e) {
+          LOG.info("show partitions: " + StringUtils.stringifyException(e));
+          return 1;
+        } catch (IOException e) {
+          LOG.info("show partitions: " + StringUtils.stringifyException(e));
+          return 1;
+        }
+        return 0;
+      }
+
+    }
+    catch (InvalidTableException e) {
+      console.printError("Table " + e.getTableName() + " does not exist");
+      LOG.debug(StringUtils.stringifyException(e));
+      return 1;
+    }
+    catch (HiveException e) {
       console.printError("FAILED: Error in metadata: " + e.getMessage(), "\n" + StringUtils.stringifyException(e));
       LOG.debug(StringUtils.stringifyException(e));
       return 1;

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java

@@ -27,12 +27,13 @@ import java.net.URLDecoder;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.lang.StringUtils;
 
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.io.*;
 import org.apache.hadoop.mapred.*;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.plan.mapredWork;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -42,6 +43,7 @@ import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
 public class ExecDriver extends Task<mapredWork> implements Serializable {
 
   private static final long serialVersionUID = 1L;
+  public static final long LOAD_PER_REDUCER = 1024 * 1024 * 1024;
 
   transient protected JobConf job;
 
@@ -80,7 +82,48 @@ public class ExecDriver extends Task<mapredWork> implements Serializable {
         LOG.warn("Number of reduce tasks not specified. Defaulting to jobconf value of: " + job.getNumReduceTasks());
         work.setNumReduceTasks(job.getNumReduceTasks());
       }
-    } 
+    }
+    else
+      LOG.info("Number of reduce tasks determined at compile : " + work.getNumReduceTasks());
+  }
+
+  /**
+   * A list of the currently running jobs spawned in this Hive instance that is used
+   * to kill all running jobs in the event of an unexpected shutdown - i.e., the JVM shuts
+   * down while there are still jobs running.
+   */
+  public static HashMap<String,String> runningJobKillURIs = new HashMap<String, String> ();
+
+
+  /**
+   * In Hive, when the user control-c's the command line, any running jobs spawned from that command 
+   * line are best-effort killed.
+   *
+   * This static constructor registers a shutdown thread to iterate over all the running job
+   * kill URLs and do a get on them.
+   *
+   */
+  static {
+    if(new org.apache.hadoop.conf.Configuration().getBoolean("webinterface.private.actions", false)) {
+      Runtime.getRuntime().addShutdownHook(new Thread() {
+          public void run() {
+            for(Iterator<String> elems = runningJobKillURIs.values().iterator(); elems.hasNext() ;  ) {
+              String uri = elems.next();
+              try {
+                System.err.println("killing job with: " + uri);
+                int retCode = ((java.net.HttpURLConnection)new java.net.URL(uri).openConnection()).getResponseCode();
+                if(retCode != 200) {
+                  System.err.println("Got an error trying to kill job with URI: " + uri + " = " + retCode);
+                }
+              } catch(Exception e) {
+                System.err.println("trying to kill job, caught: " + e);
+                // do nothing 
+              }
+            }
+          }
+        }
+                                           );
+    }
   }
 
   /**
@@ -123,6 +166,33 @@ public class ExecDriver extends Task<mapredWork> implements Serializable {
     return rj;
   }
 
+  private void inferNumReducers() throws Exception {
+    FileSystem fs = FileSystem.get(job);
+    
+    if ((work.getReducer() != null) && (work.getInferNumReducers() == true)) {
+      long inpSz = 0;
+      
+      // based on the input size - estimate the number of reducers
+      Path[] inputPaths = FileInputFormat.getInputPaths(job);
+      
+      for (Path inputP : inputPaths) {
+        if (fs.exists(inputP)) {
+          FileStatus[] fStats = fs.listStatus(inputP);
+          for (FileStatus fStat:fStats) 
+            inpSz += fStat.getLen();
+        }
+      }
+
+      
+      int newRed = (int)(inpSz / LOAD_PER_REDUCER) + 1;
+      if (newRed < work.getNumReduceTasks().intValue())
+      {
+        LOG.warn("Number of reduce tasks inferred based on input size to : " + newRed);
+        work.setNumReduceTasks(Integer.valueOf(newRed));
+      }
+    }
+  }
+
   /**
    * Execute a query plan using Hadoop
    */
@@ -141,24 +211,24 @@ public class ExecDriver extends Task<mapredWork> implements Serializable {
       LOG.info("Adding input file " + onefile);
       FileInputFormat.addInputPaths(job, onefile);
     }
-    
+        
     String hiveScratchDir = HiveConf.getVar(job, HiveConf.ConfVars.SCRATCHDIR);
     String jobScratchDir = hiveScratchDir + Utilities.randGen.nextInt();
     FileOutputFormat.setOutputPath(job, new Path(jobScratchDir));
     job.setMapperClass(ExecMapper.class);
-
+    
     job.setMapOutputValueClass(Text.class);
     job.setMapOutputKeyClass(HiveKey.class);    
-
+    
     job.setNumReduceTasks(work.getNumReduceTasks().intValue());
     job.setReducerClass(ExecReducer.class);
-
+    
     job.setInputFormat(org.apache.hadoop.hive.ql.io.HiveInputFormat.class);
-
+    
     // No-Op - we don't really write anything here .. 
     job.setOutputKeyClass(Text.class);
     job.setOutputValueClass(Text.class);
-
+    
     String auxJars = HiveConf.getVar(job, HiveConf.ConfVars.HIVEAUXJARS);
     if (StringUtils.isNotBlank(auxJars)) {
       LOG.info("adding libjars: " + auxJars);
@@ -168,15 +238,41 @@ public class ExecDriver extends Task<mapredWork> implements Serializable {
     int returnVal = 0;
     FileSystem fs = null;
     RunningJob rj = null;
-
+    
     try {
       fs = FileSystem.get(job);
+      
+      // if the input is empty exit gracefully
+      Path[] inputPaths = FileInputFormat.getInputPaths(job);
+      boolean emptyInput = true;
+      for (Path inputP : inputPaths) {
+        if(!fs.exists(inputP))
+          continue;
+        
+        FileStatus[] fStats = fs.listStatus(inputP);
+        for (FileStatus fStat:fStats) {
+        	if (fStat.getLen() > 0) {
+        	  emptyInput = false;
+        		break;
+        	}
+        }
+      }
+      	
+      if (emptyInput) {
+        console.printInfo("Job need not be submitted: no output: Success");
+      	return 0;
+      }
+      
+      inferNumReducers();
       JobClient jc = new JobClient(job);
       rj = jc.submitJob(job);
 
+      // add to list of running jobs so in case of abnormal shutdown can kill it.
+      runningJobKillURIs.put(rj.getJobID(),  rj.getTrackingURL() + "&action=kill");
+
       jobInfo(rj);
       rj = jobProgress(jc, rj);
-      
+
       String statusMesg = "Ended Job = " + rj.getJobID();
       if(!rj.isSuccessful()) {
         statusMesg += " with errors";
@@ -203,6 +299,7 @@ public class ExecDriver extends Task<mapredWork> implements Serializable {
         if(returnVal != 0 && rj != null) {
           rj.killJob();
         }
+        runningJobKillURIs.remove(rj.getJobID());
       } catch (Exception e) {}
     }
     return (returnVal);
@@ -297,5 +394,16 @@ public class ExecDriver extends Task<mapredWork> implements Serializable {
     }
     return sb.toString();
   }
+
+  @Override
+  public boolean isMapRedTask() {
+    return true;
+  }
+
+  @Override
+  public boolean hasReduce() {
+    mapredWork w = getWork();
+    return w.getReducer() != null;
+  }
 }
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java

@@ -37,7 +37,8 @@ public class ExecMapper extends MapReduceBase implements Mapper {
   private boolean abort = false;
   private Reporter rp;
   public static final Log l4j = LogFactory.getLog("ExecMapper");
-
+  private static boolean done;
+  
   public void configure(JobConf job) {
     jc = job;
     mapredWork mrwork = Utilities.getMapRedWork(job);
@@ -63,8 +64,11 @@ public class ExecMapper extends MapReduceBase implements Mapper {
     }
 
     try {
-      // Since there is no concept of a group, we don't invoke startGroup/endGroup for a mapper
-      mo.process((Writable)value);
+      if (mo.getDone())
+        done = true;
+      else
+        // Since there is no concept of a group, we don't invoke startGroup/endGroup for a mapper
+        mo.process((Writable)value);
     } catch (HiveException e) {
       abort = true;
       e.printStackTrace();
@@ -73,6 +77,19 @@ public class ExecMapper extends MapReduceBase implements Mapper {
   }
 
   public void close() {
+    // No row was processed
+    if(oc == null) {
+      try {
+        l4j.trace("Close called no row");
+        mo.initialize(jc);
+        rp = null;
+      } catch (HiveException e) {
+        abort = true;
+        e.printStackTrace();
+        throw new RuntimeException ("Map operator close failed during initialize", e);
+      }
+    }
+
     // detecting failed executions by exceptions thrown by the operator tree
     // ideally hadoop should let us know whether map execution failed or not
     try {
@@ -89,6 +106,10 @@ public class ExecMapper extends MapReduceBase implements Mapper {
     }
   }
 
+  public static boolean getDone() {
+    return done;
+  }
+
   public static class reportStats implements Operator.OperatorFunc {
     Reporter rp;
     public reportStats (Reporter rp) {

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java

@@ -183,6 +183,20 @@ public class ExecReducer extends MapReduceBase implements Reducer {
   }
 
   public void close() {
+
+    // No row was processed
+    if(oc == null) {
+      try {
+        l4j.trace("Close called no row");
+        reducer.initialize(jc);
+        rp = null;
+      } catch (HiveException e) {
+        abort = true;
+        e.printStackTrace();
+        throw new RuntimeException ("Reduce operator close failed during initialize", e);
+      }
+    }
+
     try {
       if (groupKey != null) {
         // If a operator wants to do some work at the end of a group

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java

@@ -28,6 +28,8 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Arrays;
+import java.util.Comparator;
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hive.ql.plan.explain;
@@ -183,7 +185,10 @@ public class ExplainTask extends Task<explainWork> implements Serializable {
     }
     
     // We look at all methods that generate values for explain
-    for(Method m: work.getClass().getMethods()) {
+    Method[] methods = work.getClass().getMethods();
+    Arrays.sort(methods, new MethodComparator());
+
+    for(Method m: methods) {
       int prop_indents = indent+2;
       note = m.getAnnotation(explain.class);
 
@@ -330,4 +335,13 @@ public class ExplainTask extends Task<explainWork> implements Serializable {
                  new HashSet<Task<? extends Serializable>>(), indent+2);
     }
   }
+
+  public static class MethodComparator implements Comparator {
+    public int compare(Object o1, Object o2) {
+      Method m1 = (Method)o1;
+      Method m2 = (Method)o2;
+      return m1.getName().compareTo(m2.getName());
+    }
+  }
+
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java

@@ -18,11 +18,16 @@
 
 package org.apache.hadoop.hive.ql.exec;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc;
 
 import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 
@@ -31,9 +36,10 @@ public class ExprNodeFieldEvaluator extends ExprNodeEvaluator {
   protected exprNodeFieldDesc desc;
   transient ExprNodeEvaluator leftEvaluator;
   transient InspectableObject leftInspectableObject;
-  transient StructObjectInspector cachedLeftObjectInspector;
+  transient StructObjectInspector structObjectInspector;
   transient StructField field;
-  transient ObjectInspector fieldObjectInspector;
+  transient ObjectInspector structFieldObjectInspector;
+  transient ObjectInspector resultObjectInspector;
   
   public ExprNodeFieldEvaluator(exprNodeFieldDesc desc) {
     this.desc = desc;
@@ -50,14 +56,19 @@ public class ExprNodeFieldEvaluator extends ExprNodeEvaluator {
     leftEvaluator.evaluate(row, rowInspector, leftInspectableObject);
 
     if (field == null) {
-      cachedLeftObjectInspector = (StructObjectInspector)leftInspectableObject.oi;
-      field = cachedLeftObjectInspector.getStructFieldRef(desc.getFieldName());
-      fieldObjectInspector = field.getFieldObjectInspector();
+      evaluateInspector(rowInspector);
+    }
+    result.oi = resultObjectInspector;
+    if (desc.getIsList()) {
+      List<?> list = ((ListObjectInspector)leftInspectableObject.oi).getList(leftInspectableObject.o);
+      List<Object> r = new ArrayList<Object>(list.size());
+      for(int i=0; i<list.size(); i++) {
+        r.add(structObjectInspector.getStructFieldData(list.get(i), field));
+      }
+      result.o = r;
     } else {
-      assert(cachedLeftObjectInspector == leftInspectableObject.oi);
+      result.o = structObjectInspector.getStructFieldData(leftInspectableObject.o, field);
     }
-    result.oi = fieldObjectInspector;
-    result.o = cachedLeftObjectInspector.getStructFieldData(leftInspectableObject.o, field); 
   }
 
   public ObjectInspector evaluateInspector(ObjectInspector rowInspector)
@@ -66,13 +77,20 @@ public class ExprNodeFieldEvaluator extends ExprNodeEvaluator {
     // is different from the previous row 
     leftInspectableObject.oi = leftEvaluator.evaluateInspector(rowInspector);
     if (field == null) {
-      cachedLeftObjectInspector = (StructObjectInspector)leftInspectableObject.oi;
-      field = cachedLeftObjectInspector.getStructFieldRef(desc.getFieldName());
-      fieldObjectInspector = field.getFieldObjectInspector();
+      if (desc.getIsList()) {
+        structObjectInspector = (StructObjectInspector)((ListObjectInspector)leftInspectableObject.oi).getListElementObjectInspector();
+      } else {
+        structObjectInspector = (StructObjectInspector)leftInspectableObject.oi;
+      }
+      field = structObjectInspector.getStructFieldRef(desc.getFieldName());
+      structFieldObjectInspector = field.getFieldObjectInspector();
+    }
+    if (desc.getIsList()) {
+      resultObjectInspector = ObjectInspectorFactory.getStandardListObjectInspector(structFieldObjectInspector);
     } else {
-      assert(cachedLeftObjectInspector == leftInspectableObject.oi);      
+      resultObjectInspector = structFieldObjectInspector;
     }
-    return fieldObjectInspector;
+    return resultObjectInspector;
   }
 
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java

@@ -22,7 +22,9 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.exprNodeIndexDesc;
 import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
 import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
 
 public class ExprNodeIndexEvaluator extends ExprNodeEvaluator {
 
@@ -44,16 +46,38 @@ public class ExprNodeIndexEvaluator extends ExprNodeEvaluator {
     assert(result != null);
     mainEvaluator.evaluate(row, rowInspector, mainInspectableObject);
     indexEvaluator.evaluate(row, rowInspector, indexInspectableObject);
-    int index = ((Number)indexInspectableObject.o).intValue();
+
+    if (mainInspectableObject.oi.getCategory() == Category.LIST) {
+      int index = ((Number)indexInspectableObject.o).intValue();
     
-    ListObjectInspector loi = (ListObjectInspector)mainInspectableObject.oi;
-    result.oi = loi.getListElementObjectInspector();
-    result.o = loi.getListElement(mainInspectableObject.o, index);
+      ListObjectInspector loi = (ListObjectInspector)mainInspectableObject.oi;
+      result.oi = loi.getListElementObjectInspector();
+      result.o = loi.getListElement(mainInspectableObject.o, index);
+    }
+    else if (mainInspectableObject.oi.getCategory() == Category.MAP) {
+      MapObjectInspector moi = (MapObjectInspector)mainInspectableObject.oi;
+      result.oi = moi.getMapValueObjectInspector();
+      result.o = moi.getMapValueElement(mainInspectableObject.o, indexInspectableObject.o);
+    }
+    else {
+      // Should never happen because we checked this in SemanticAnalyzer.getXpathOrFuncExprNodeDesc
+      throw new RuntimeException("Hive 2 Internal error: cannot evaluate index expression on "
+          + mainInspectableObject.oi.getTypeName());
+    }
   }
 
   public ObjectInspector evaluateInspector(ObjectInspector rowInspector)
       throws HiveException {
-    return ((ListObjectInspector)mainEvaluator.evaluateInspector(rowInspector)).getListElementObjectInspector();
+    ObjectInspector mainInspector = mainEvaluator.evaluateInspector(rowInspector);
+    if (mainInspector.getCategory() == Category.LIST) {
+      return ((ListObjectInspector)mainInspector).getListElementObjectInspector();
+    } else if (mainInspector.getCategory() == Category.MAP) {
+      return ((MapObjectInspector)mainInspector).getMapValueObjectInspector();
+    } else {
+      // Should never happen because we checked this in SemanticAnalyzer.getXpathOrFuncExprNodeDesc
+      throw new RuntimeException("Hive 2 Internal error: cannot evaluate index expression on "
+          + mainInspector.getTypeName());
+    }
   }
 
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java

@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import java.io.Serializable;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Vector;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.plan.fetchWork;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * FetchTask implementation
+ **/
+public class FetchTask extends Task<fetchWork> implements Serializable {
+  private static final long serialVersionUID = 1L;
+
+  static final private int MAX_ROWS  = 100;
+  
+  public void initialize (HiveConf conf) {
+   	super.initialize(conf);
+    splitNum = 0;
+    currRecReader = null;
+    
+   	try {
+       // Create a file system handle
+       fs = FileSystem.get(conf);   
+       serde = work.getDeserializerClass().newInstance();
+       serde.initialize(null, work.getSchema());
+       job = new JobConf(conf, ExecDriver.class);
+       Path inputP = work.getSrcDir();
+       if(!fs.exists(inputP)) {
+         empty = true;
+         return;
+       }
+
+       empty = true;
+       FileStatus[] fStats = fs.listStatus(inputP);
+       for (FileStatus fStat:fStats) {
+         if (fStat.getLen() > 0) {
+           empty = false;
+           break;
+         }
+       }
+
+       if (empty)
+         return;
+
+       FileInputFormat.setInputPaths(job, inputP);
+       inputFormat = getInputFormatFromCache(work.getInputFormatClass(), job);
+	     inputSplits = inputFormat.getSplits(job, 1);
+	 	   mSerde = new MetadataTypedColumnsetSerDe();
+       Properties mSerdeProp = new Properties();
+       mSerdeProp.put(Constants.SERIALIZATION_FORMAT, "" + Utilities.tabCode);
+       mSerdeProp.put(Constants.SERIALIZATION_NULL_FORMAT, "NULL");
+       mSerde.initialize(null, mSerdeProp);
+       totalRows = 0;
+    } catch (Exception e) {
+      // Bail out ungracefully - we should never hit
+      // this here - but would have hit it in SemanticAnalyzer
+      LOG.error(StringUtils.stringifyException(e));
+      throw new RuntimeException (e);
+    }
+  }
+  
+  public int execute() {
+  	assert false;
+  	return 0;
+  }
+  
+  /**
+   * A cache of InputFormat instances.
+   */
+  private static Map<Class, InputFormat<WritableComparable, Writable>> inputFormats =
+    new HashMap<Class, InputFormat<WritableComparable, Writable>>();
+  
+  static InputFormat<WritableComparable, Writable> getInputFormatFromCache(Class inputFormatClass, Configuration conf) throws IOException {
+    if (!inputFormats.containsKey(inputFormatClass)) {
+      try {
+        InputFormat<WritableComparable, Writable> newInstance =
+          (InputFormat<WritableComparable, Writable>)ReflectionUtils.newInstance(inputFormatClass, conf);
+        inputFormats.put(inputFormatClass, newInstance);
+      } catch (Exception e) {
+        throw new IOException("Cannot create an instance of InputFormat class " + inputFormatClass.getName()
+                               + " as specified in mapredWork!");
+      }
+    }
+    return inputFormats.get(inputFormatClass);
+  }
+  
+  private int splitNum;
+  private FileSystem fs;  
+  private RecordReader<WritableComparable, Writable> currRecReader;
+  private InputSplit[] inputSplits;
+  private InputFormat  inputFormat;
+  private JobConf      job;
+	private WritableComparable key; 
+	private Writable value;
+	private Deserializer  serde;
+	private MetadataTypedColumnsetSerDe mSerde;
+	private int totalRows;
+  private boolean empty;
+  
+ 	private RecordReader<WritableComparable, Writable> getRecordReader() throws Exception {
+		if (splitNum >= inputSplits.length) 
+  	  return null;
+		currRecReader = inputFormat.getRecordReader(inputSplits[splitNum++], job, Reporter.NULL);
+		key = currRecReader.createKey();
+		value = currRecReader.createValue();
+		return currRecReader;
+	}
+ 	
+  public boolean fetch(Vector<String> res) {
+  	try {
+      if (empty)
+        return false;
+
+      int numRows = 0;
+      int rowsRet = MAX_ROWS;
+      if ((work.getLimit() >= 0) && ((work.getLimit() - totalRows) < rowsRet))
+        rowsRet = work.getLimit() - totalRows;
+      if (rowsRet <= 0) {
+        if (currRecReader != null)
+          currRecReader.close();
+        return false;
+      }
+
+    	while (numRows < rowsRet) {
+  	    if (currRecReader == null) {
+  	  	  currRecReader = getRecordReader();
+  	  		if (currRecReader == null) {
+            if (numRows == 0) 
+            	return false;
+            totalRows += numRows;
+            return true;
+    	    }
+  	    }
+      	boolean ret = currRecReader.next(key, value);
+   	  	if (ret) {
+   	  		Object obj = serde.deserialize(value);
+   	  		res.add(((Text)mSerde.serialize(obj, serde.getObjectInspector())).toString());
+   	  		numRows++;
+   	  	}
+   	  	else {
+          currRecReader.close();
+   	  		currRecReader = getRecordReader();
+   	  		if (currRecReader == null) {
+            if (numRows == 0) 
+            	return false;
+            totalRows += numRows;
+            return true;
+    	    }
+          else {
+        		key = currRecReader.createKey();
+        		value = currRecReader.createValue();
+          }
+      	}
+      }
+    	totalRows += numRows;
+      return true;
+    }
+    catch (Exception e) {
+      console.printError("Failed with exception " +   e.getMessage(), "\n" + StringUtils.stringifyException(e));
+      return false;
+    }
+  }
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java

@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.fileSinkDesc;
 import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.serde.Constants;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.Serializer;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -101,12 +102,27 @@ public class FileSinkOperator extends TerminalOperator <fileSinkDesc> implements
         if(isCompressed) {
           finalPath = new Path(conf.getDirName(), Utilities.getTaskId(hconf) + ".gz");
         }
+        String rowSeparatorString = conf.getTableInfo().getProperties().getProperty(Constants.LINE_DELIM, "\n");
+        int rowSeparator = 0;
+        try {
+          rowSeparator = Byte.parseByte(rowSeparatorString); 
+        } catch (NumberFormatException e) {
+          rowSeparator = rowSeparatorString.charAt(0); 
+        }
+        final int finalRowSeparator = rowSeparator;  
         final OutputStream outStream = Utilities.createCompressedStream(jc, fs.create(outPath));
         outWriter = new RecordWriter () {
             public void write(Writable r) throws IOException {
-              Text tr = (Text)r;
-              outStream.write(tr.getBytes(), 0, tr.getLength());
-              outStream.write('\n');
+              if (r instanceof Text) {
+                Text tr = (Text)r;
+                outStream.write(tr.getBytes(), 0, tr.getLength());
+                outStream.write(finalRowSeparator);
+              } else {
+                // DynamicSerDe always writes out BytesWritable
+                BytesWritable bw = (BytesWritable)r;
+                outStream.write(bw.get(), 0, bw.getSize());
+                outStream.write(finalRowSeparator);
+              }
             }
             public void close(boolean abort) throws IOException {
               outStream.close();

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java

@@ -52,7 +52,15 @@ public class FunctionRegistry {
     registerUDF("str_lt", UDFStrLt.class, OperatorType.PREFIX, false);
     registerUDF("str_ge", UDFStrGe.class, OperatorType.PREFIX, false);
     registerUDF("str_le", UDFStrLe.class, OperatorType.PREFIX, false);
+    
+    registerUDF("size", UDFSize.class, OperatorType.PREFIX, false);
 
+    registerUDF("round", UDFRound.class, OperatorType.PREFIX, false);
+    registerUDF("floor", UDFFloor.class, OperatorType.PREFIX, false);
+    registerUDF("ceil", UDFCeil.class, OperatorType.PREFIX, false);
+    registerUDF("ceiling", UDFCeil.class, OperatorType.PREFIX, false);
+    registerUDF("rand", UDFRand.class, OperatorType.PREFIX, false);
+    
     registerUDF("upper", UDFUpper.class, OperatorType.PREFIX, false);
     registerUDF("lower", UDFLower.class, OperatorType.PREFIX, false);
     registerUDF("ucase", UDFUpper.class, OperatorType.PREFIX, false);
@@ -66,6 +74,9 @@ public class FunctionRegistry {
     registerUDF("regexp", UDFRegExp.class, OperatorType.INFIX, true);
     registerUDF("regexp_replace", UDFRegExpReplace.class, OperatorType.PREFIX, false);
 
+    registerUDF("positive", UDFOPPositive.class, OperatorType.PREFIX, true, "+");
+    registerUDF("negative", UDFOPNegative.class, OperatorType.PREFIX, true, "-");
+
     registerUDF("+", UDFOPPlus.class, OperatorType.INFIX, true);
     registerUDF("-", UDFOPMinus.class, OperatorType.INFIX, true);
     registerUDF("*", UDFOPMultiply.class, OperatorType.INFIX, true);
@@ -75,7 +86,7 @@ public class FunctionRegistry {
     registerUDF("&", UDFOPBitAnd.class, OperatorType.INFIX, true);
     registerUDF("|", UDFOPBitOr.class, OperatorType.INFIX, true);
     registerUDF("^", UDFOPBitXor.class, OperatorType.INFIX, true);
-    registerUDF("~", UDFOPBitNot.class, OperatorType.INFIX, true);
+    registerUDF("~", UDFOPBitNot.class, OperatorType.PREFIX, true);
 
     registerUDF("=", UDFOPEqual.class, OperatorType.INFIX, true);
     registerUDF("==", UDFOPEqual.class, OperatorType.INFIX, true, "=");
@@ -89,8 +100,8 @@ public class FunctionRegistry {
     registerUDF("&&", UDFOPAnd.class, OperatorType.INFIX, true, "and");
     registerUDF("or", UDFOPOr.class, OperatorType.INFIX, true);
     registerUDF("||", UDFOPOr.class, OperatorType.INFIX, true, "or");
-    registerUDF("not", UDFOPNot.class, OperatorType.INFIX, true);
-    registerUDF("!", UDFOPNot.class, OperatorType.INFIX, true, "not");
+    registerUDF("not", UDFOPNot.class, OperatorType.PREFIX, true);
+    registerUDF("!", UDFOPNot.class, OperatorType.PREFIX, true, "not");
 
     registerUDF("isnull", UDFOPNull.class, OperatorType.POSTFIX, true, "is null");
     registerUDF("isnotnull", UDFOPNotNull.class, OperatorType.POSTFIX, true, "is not null");

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java

@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import java.io.*;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.limitDesc;
+import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Limit operator implementation
+ * Limits a subobject and passes that on.
+ **/
+public class LimitOperator extends Operator<limitDesc> implements Serializable {
+  private static final long serialVersionUID = 1L;
+  
+  transient protected int limit;
+  transient protected int currCount;
+
+  public void initialize(Configuration hconf) throws HiveException {
+    super.initialize(hconf);
+    limit = conf.getLimit();
+    currCount = 0;
+  }
+
+  public void process(Object row, ObjectInspector rowInspector) throws HiveException {
+    if (currCount < limit) {
+      forward(row, rowInspector);
+      currCount++;
+    }
+    else
+      setDone(true);
+  }
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java

@@ -88,4 +88,15 @@ public class MapRedTask extends Task<mapredWork> implements Serializable {
       return (1);
     }
   }
+
+  @Override
+  public boolean isMapRedTask() {
+    return true;
+  }
+
+  @Override
+  public boolean hasReduce() {
+    mapredWork w = getWork();
+    return w.getReducer() != null;
+  }
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java

@@ -55,6 +55,7 @@ public abstract class Operator <T extends Serializable> implements Serializable
 
   protected String id;
   protected T conf;
+  protected boolean done;
 
   public void setConf(T conf) {
     this.conf = conf;
@@ -73,6 +74,14 @@ public abstract class Operator <T extends Serializable> implements Serializable
     return id;
   }
 
+  public boolean getDone() {
+    return done;
+  }
+
+  public void setDone(boolean done) {
+    this.done = done;
+  }
+
   // non-bean fields needed during compilation
   transient private RowSchema rowSchema;
 
@@ -219,9 +228,24 @@ public abstract class Operator <T extends Serializable> implements Serializable
 
   protected void forward(Object row, ObjectInspector rowInspector) throws HiveException {
     
-    if(childOperators == null) {
+    if((childOperators == null) || (getDone())) {
       return;
     }
+    
+    // if all children are done, this operator is also done
+    boolean isDone = true;
+    for(Operator<? extends Serializable> o: childOperators) {
+      if (!o.getDone()) {
+        isDone = false;
+        break;
+      }
+    }
+
+    if (isDone) {
+      setDone(isDone);
+      return;
+    }
+
     for(Operator<? extends Serializable> o: childOperators) {
       o.process(row, rowInspector);
     }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java

@@ -48,6 +48,7 @@ public class OperatorFactory {
     opvec.add(new opTuple<extractDesc> (extractDesc.class, ExtractOperator.class));
     opvec.add(new opTuple<groupByDesc> (groupByDesc.class, GroupByOperator.class));
     opvec.add(new opTuple<joinDesc> (joinDesc.class, JoinOperator.class));
+    opvec.add(new opTuple<limitDesc> (limitDesc.class, LimitOperator.class));
   }
               
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java

@@ -134,10 +134,10 @@ public class ReduceSinkOperator extends TerminalOperator <reduceSinkDesc> implem
       }
       keyWritable.setHashCode(keyHashCode);
       
-      ArrayList<String> values = new ArrayList<String>(valueEval.length);
+      ArrayList<Object> values = new ArrayList<Object>(valueEval.length);
       for(ExprNodeEvaluator e: valueEval) {
         e.evaluate(row, rowInspector, tempInspectableObject);
-        values.add(tempInspectableObject.o == null ? null : tempInspectableObject.o.toString());
+        values.add(tempInspectableObject.o);
         if (valueObjectInspector == null) {
           valueFieldsObjectInspectors.add(tempInspectableObject.oi);
         }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java

@@ -86,6 +86,12 @@ public abstract class Task <T extends Serializable> implements Serializable {
   }
 
   public abstract int execute();
+  
+  // dummy method - FetchTask overwrites this
+  public boolean fetch(Vector<String> res) { 
+    assert false;
+  	return false;
+  }
 
   public void setChildTasks(List<Task<? extends Serializable>> childTasks) {
     this.childTasks = childTasks;
@@ -158,4 +164,11 @@ public abstract class Task <T extends Serializable> implements Serializable {
     return id;
   }
 
+  public boolean isMapRedTask() {
+    return false;
+  }
+
+  public boolean hasReduce() {
+    return false;
+  }
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java

@@ -44,6 +44,7 @@ public class TaskFactory {
     id = 0;
     taskvec = new ArrayList<taskTuple<? extends Serializable>>();
     taskvec.add(new taskTuple<moveWork>(moveWork.class, MoveTask.class));
+    taskvec.add(new taskTuple<fetchWork>(fetchWork.class, FetchTask.class));
     taskvec.add(new taskTuple<copyWork>(copyWork.class, CopyTask.class));
     taskvec.add(new taskTuple<DDLWork>(DDLWork.class, DDLTask.class));
     taskvec.add(new taskTuple<FunctionWork>(FunctionWork.class, FunctionTask.class));

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/UDF.java

@@ -19,30 +19,20 @@
 package org.apache.hadoop.hive.ql.exec;
 
 /**
- * A dummy User-defined function (UDF) for the use with Hive.
+ * A User-defined function (UDF) for the use with Hive.
  * 
- * New UDF classes do NOT need to inherit from this UDF class.
+ * New UDF classes need to inherit from this UDF class.
  * 
  * Required for all UDF classes:
- * 1. Implement a single method named "evaluate" which will be called by Hive.
+ * 1. Implement one or more methods named "evaluate" which will be called by Hive.
  *    The following are some examples:
+ *    public int evaluate();
  *    public int evaluate(int a);
  *    public double evaluate(int a, double b);
  *    public String evaluate(String a, int b, String c);
  * 
- *    "evaluate" should neither be a void method, nor should it returns "null" in any case.
- *    In both cases, the Hive system will throw an HiveException saying the evaluation of UDF
- *    is failed.
+ *    "evaluate" should never be a void method.  However it can return "null" if needed.
  */
-public class UDF {
+public interface UDF {
 
-  public UDF() { }
-  
-  /** Evaluate the UDF.
-   *  @return plain old java object
-   **/
-  public int evaluate() {
-    return 0;
-  }
-  
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java

@@ -382,7 +382,7 @@ public class Utilities {
   }
 
   public final static String NSTR = "";
-  public static enum streamStatus {EOF, TERMINATED, NORMAL}
+  public static enum streamStatus {EOF, TERMINATED}
   public static streamStatus readColumn(DataInput in, OutputStream out) throws IOException {
 
     while (true) {
@@ -397,10 +397,6 @@ public class Utilities {
         return streamStatus.TERMINATED;
       }
 
-      if (b == Utilities.ctrlaCode) {
-        return streamStatus.NORMAL;
-      }
-
       out.write(b);
     }
     // Unreachable

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java

@@ -179,8 +179,8 @@ public class HiveInputFormat<K extends WritableComparable,
     }
 
     InputFormat inputFormat = getInputFormatFromCache(inputFormatClass);
-
-    return inputFormat.getRecordReader(inputSplit, job, reporter);
+    
+    return new HiveRecordReader(inputFormat.getRecordReader(inputSplit, job, reporter));
   }
 
 
@@ -219,6 +219,7 @@ public class HiveInputFormat<K extends WritableComparable,
     return result.toArray(new HiveInputSplit[result.size()]);
   }
 
+
   private tableDesc getTableDescFromPath(Path dir) throws IOException {
 
     partitionDesc partDesc = pathToPartitionInfo.get(dir.toString());

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/io/HiveRecordReader.java

@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io;
+
+import org.apache.hadoop.hive.ql.exec.ExecMapper;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import java.io.IOException;
+
+public class HiveRecordReader<K extends WritableComparable, V extends Writable>  
+  implements RecordReader<K, V> {
+
+  private RecordReader recordReader;
+  public HiveRecordReader(RecordReader recordReader){ 
+    this.recordReader = recordReader;
+  }
+
+  public void close() throws IOException { 
+    recordReader.close(); 
+  }
+
+  public K createKey() { 
+    return (K)recordReader.createKey();
+  }
+
+  public V createValue() { 
+    return (V)recordReader.createValue();
+  }
+
+  public long getPos() throws IOException { 
+    return recordReader.getPos();
+  }
+
+  public float getProgress() throws IOException { 
+    return recordReader.getProgress();
+  }
+  
+  public boolean  next(K key, V value) throws IOException { 
+    if (ExecMapper.getDone())
+      return false;
+    return recordReader.next(key, value);
+  }
+}
+

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

@@ -105,7 +105,6 @@ public class Hive {
     this.conf = c;
     try {
        msc = this.createMetaStoreClient();
-       //msc = new HiveMetaStoreClient(this.conf);
     } catch (MetaException e) {
       throw new HiveException("Unable to open connection to metastore", e);
     }
@@ -169,7 +168,7 @@ public class Hive {
         tbl.getPartCols().add(part);
       }
     }
-    tbl.setSerializationLib(org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.shortName());
+    tbl.setSerializationLib(org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
     tbl.setNumBuckets(bucketCount);
     createTable(tbl);
   }
@@ -269,7 +268,7 @@ public class Hive {
     } catch (NoSuchObjectException e) {
       if(throwException) {
         LOG.error(StringUtils.stringifyException(e));
-        throw new InvalidTableException("Table not found " + tableName);
+        throw new InvalidTableException("Table not found ", tableName);
       }
       return null;
     } catch (Exception e) {
@@ -463,6 +462,17 @@ public class Hive {
     return new Partition(tbl, tpart);
   }
   
+  public boolean dropPartition(String db_name, String tbl_name, List<String> part_vals,
+      boolean deleteData) throws HiveException {
+    try {
+      return msc.dropPartition(db_name, tbl_name, part_vals, deleteData);
+    } catch (NoSuchObjectException e) {
+      throw new HiveException("Partition or table doesn't exist.", e);
+    } catch (Exception e) {
+      throw new HiveException("Unknow error. Please check logs.", e);
+    }
+  }
+
   public List<String> getPartitionNames(String dbName, String tblName, short max) throws HiveException {
     List names = null;
     try {

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java

@@ -58,6 +58,14 @@ public class Partition {
 
     private LinkedHashMap<String, String> spec;
     
+    /**
+     * @return
+     * @see org.apache.hadoop.hive.metastore.api.Partition#getValues()
+     */
+    public List<String> getValues() {
+      return tPartition.getValues();
+    }
+
     private Path partPath;
     private URI partURI;
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java

@@ -95,7 +95,7 @@ public class Table {
     initEmpty();
     this.schema = schema;
     this.deserializer = deserializer; //TODO: convert to SerDeInfo format
-    this.getTTable().getSd().getSerdeInfo().setSerializationLib(deserializer.getShortName());
+    this.getTTable().getSd().getSerdeInfo().setSerializationLib(deserializer.getClass().getName());
     getTTable().setTableName(name);
     getSerdeInfo().setSerializationLib(deserializer.getClass().getName());
     setInputFormatClass(inputFormatClass);
@@ -108,7 +108,7 @@ public class Table {
     initEmpty();
     getTTable().setTableName(name);
     getTTable().setDbName(MetaStoreUtils.DEFAULT_DATABASE_NAME);
-    getSerdeInfo().setSerializationLib(MetadataTypedColumnsetSerDe.shortName());
+    getSerdeInfo().setSerializationLib(MetadataTypedColumnsetSerDe.class.getName());
     getSerdeInfo().getParameters().put(Constants.SERIALIZATION_FORMAT, "1");
   }
   

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java

@@ -31,7 +31,6 @@ import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.metadata.*;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.exec.TaskFactory;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
 
 
@@ -43,6 +42,8 @@ public abstract class BaseSemanticAnalyzer {
   protected final Hive db;
   protected final HiveConf conf;
   protected List<Task<? extends Serializable>> rootTasks;
+  protected Task<? extends Serializable> fetchTask;
+  protected boolean fetchTaskInit;
   protected final Log LOG;
   protected final LogHelper console;
 
@@ -65,13 +66,40 @@ public abstract class BaseSemanticAnalyzer {
     }
   }
 
-  public abstract void analyze(CommonTree ast, Context ctx) throws SemanticException;
+  public abstract void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException;
+
+  public void analyze(CommonTree ast, Context ctx) throws SemanticException {
+    scratchDir = ctx.getScratchDir();
+    analyzeInternal(ast, ctx);
+  }
   
   public List<Task<? extends Serializable>> getRootTasks() {
     return rootTasks;
   }
 
-  protected void reset() {
+  /**
+	 * @return the fetchTask
+	 */
+	public Task<? extends Serializable> getFetchTask() {
+		return fetchTask;
+	}
+
+	/**
+	 * @param fetchTask the fetchTask to set
+	 */
+	public void setFetchTask(Task<? extends Serializable> fetchTask) {
+		this.fetchTask = fetchTask;
+	}
+
+	public boolean getFetchTaskInit() {
+		return fetchTaskInit;
+	}
+
+	public void setFetchTaskInit(boolean fetchTaskInit) {
+		this.fetchTaskInit = fetchTaskInit;
+	}
+
+	protected void reset() {
     rootTasks = new ArrayList<Task<? extends Serializable>>();
   }
 
@@ -118,9 +146,33 @@ public abstract class BaseSemanticAnalyzer {
   public static String unescapeSQLString(String b) {
     assert(b.charAt(0) == '\'');
     assert(b.charAt(b.length()-1) == '\'');
+
+    // Some of the strings can be passed in as unicode. For example, the
+    // delimiter can be passed in as \002 - So, we first check if the 
+    // string is a unicode number, else go back to the old behavior
     StringBuilder sb = new StringBuilder(b.length());
-    for(int i=1; i+1<b.length(); i++) {
-      if (b.charAt(i) == '\\' && i+2<b.length()) {
+    int i = 1;
+    while (i < (b.length()-1)) {
+
+      if (b.charAt(i) == '\\' && (i+4 < b.length())) {
+        char i1 = b.charAt(i+1);
+        char i2 = b.charAt(i+2);
+        char i3 = b.charAt(i+3);
+        if ((i1 >= '0' && i1 <= '1') &&
+            (i2 >= '0' && i2 <= '7') &&
+            (i3 >= '0' && i3 <= '7'))
+        {
+          byte bVal = (byte)((i3 - '0') + ((i2 - '0') * 8 ) + ((i1 - '0') * 8 * 8));
+          byte[] bValArr = new byte[1];
+          bValArr[0] = bVal;
+          String tmp = new String(bValArr);
+          sb.append(tmp);
+          i += 4;
+          continue;
+        }
+      }
+        
+      if (b.charAt(i) == '\\' && (i+2 < b.length())) {
         char n=b.charAt(i+1);
         switch(n) {
         case '0': sb.append("\0"); break;
@@ -141,6 +193,7 @@ public abstract class BaseSemanticAnalyzer {
       } else {
         sb.append(b.charAt(i));
       }
+      i++;
     }
     return sb.toString();
   }
@@ -159,7 +212,7 @@ public abstract class BaseSemanticAnalyzer {
     public HashMap<String, String> partSpec;
     public Partition partHandle;
 
-    public tableSpec(Hive db, CommonTree ast) throws SemanticException {
+    public tableSpec(Hive db, CommonTree ast, boolean forceCreatePartition) throws SemanticException {
 
       assert(ast.getToken().getType() == HiveParser.TOK_TAB);
       int childIndex = 0;
@@ -179,7 +232,10 @@ public abstract class BaseSemanticAnalyzer {
             String val = stripQuotes(partspec_val.getChild(1).getText());
             partSpec.put(partspec_val.getChild(0).getText(), val);
           }
-          partHandle = Hive.get().getPartition(tableHandle, partSpec, true);
+          partHandle = Hive.get().getPartition(tableHandle, partSpec, forceCreatePartition);
+          if(partHandle == null) {
+            throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(ast.getChild(childIndex)));
+          }
         }
       } catch (InvalidTableException ite) {
         throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(ast.getChild(0)), ite);

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java

@@ -18,13 +18,22 @@
 
 package org.apache.hadoop.hive.ql.parse;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.antlr.runtime.tree.CommonTree;
+import org.antlr.runtime.tree.Tree;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Order;
-
-import org.antlr.runtime.tree.CommonTree;
-
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.exec.TaskFactory;
 import org.apache.hadoop.hive.ql.plan.DDLWork;
@@ -32,17 +41,11 @@ import org.apache.hadoop.hive.ql.plan.alterTableDesc;
 import org.apache.hadoop.hive.ql.plan.createTableDesc;
 import org.apache.hadoop.hive.ql.plan.descTableDesc;
 import org.apache.hadoop.hive.ql.plan.dropTableDesc;
+import org.apache.hadoop.hive.ql.plan.showPartitionsDesc;
 import org.apache.hadoop.hive.ql.plan.showTablesDesc;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.alterTableDesc.alterTableTypes;
 import org.apache.hadoop.hive.serde.Constants;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import java.util.*;
-
 public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
   private static final Log LOG = LogFactory.getLog("hive.ql.parse.DDLSemanticAnalyzer");
   public static final Map<Integer, String> TokenToTypeName = new HashMap<Integer, String>();
@@ -67,7 +70,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
   }
 
   @Override
-  public void analyze(CommonTree ast, Context ctx) throws SemanticException {
+  public void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException {
     this.ctx = ctx;
     if (ast.getToken().getType() == HiveParser.TOK_CREATETABLE)
        analyzeCreateTable(ast, false);
@@ -88,7 +91,16 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
     else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_RENAME)
       analyzeAlterTableRename(ast);
     else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_ADDCOLS)
-      analyzeAlterTableAddCols(ast);
+      analyzeAlterTableModifyCols(ast, alterTableTypes.ADDCOLS);
+    else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_REPLACECOLS)
+      analyzeAlterTableModifyCols(ast, alterTableTypes.REPLACECOLS);
+    else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_DROPPARTS)
+      analyzeAlterTableDropParts(ast);
+    else if (ast.getToken().getType() == HiveParser.TOK_SHOWPARTITIONS)
+    {
+      ctx.setResFile(new Path(getTmpFileName()));
+      analyzeShowPartitions(ast);
+    }
   }
 
   private void analyzeCreateTable(CommonTree ast, boolean isExt) 
@@ -107,8 +119,10 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
     String            comment       = null;
     boolean           isSequenceFile  = false;
     String            location      = null;
+    String            serde         = null;
+    Map<String, String> mapProp     = null;
 
-    LOG.info("Creating table" + tableName);
+    LOG.info("Creating table" + tableName);    
     int numCh = ast.getChildCount();
     for (int num = 2; num < numCh; num++)
     {
@@ -152,6 +166,18 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
             }
           }
           break;
+        case HiveParser.TOK_TABLESERIALIZER:
+          serde = unescapeSQLString(child.getChild(0).getText());
+          if (child.getChildCount() == 2) {
+            mapProp = new HashMap<String, String>();
+            CommonTree prop = (CommonTree)((CommonTree)child.getChild(1)).getChild(0);
+            for (int propChild = 0; propChild < prop.getChildCount(); propChild++) {
+              String key = unescapeSQLString(prop.getChild(propChild).getChild(0).getText());
+              String value = unescapeSQLString(prop.getChild(propChild).getChild(1).getText());
+              mapProp.put(key,value);
+            }
+          }
+          break;
         case HiveParser.TOK_TBLSEQUENCEFILE:
           isSequenceFile = true;
           break;
@@ -166,7 +192,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
       new createTableDesc(tableName, isExt, cols, partCols, bucketCols, 
                           sortCols, numBuckets,
                           fieldDelim, collItemDelim, mapKeyDelim, lineDelim,
-                          comment, isSequenceFile, location);
+                          comment, isSequenceFile, location, serde, mapProp);
 
     validateCreateTable(crtTblDesc);
     rootTasks.add(TaskFactory.get(new DDLWork(crtTblDesc), conf));
@@ -310,12 +336,34 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
   
   private void analyzeDescribeTable(CommonTree ast) 
   throws SemanticException {
-    String tableName = ast.getChild(0).getText();    
-    descTableDesc descTblDesc = new descTableDesc(ctx.getResFile(), tableName);
+    Tree table_t = ast.getChild(0);
+    String tableName = table_t.getChild(0).getText();
+    HashMap<String, String> partSpec = null;
+    // get partition metadata if partition specified
+    if (table_t.getChildCount() == 2) {
+      CommonTree partspec = (CommonTree) table_t.getChild(1);
+      partSpec = new LinkedHashMap<String, String>();
+      for (int i = 0; i < partspec.getChildCount(); ++i) {
+        CommonTree partspec_val = (CommonTree) partspec.getChild(i);
+        String val = stripQuotes(partspec_val.getChild(1).getText());
+        partSpec.put(partspec_val.getChild(0).getText(), val);
+      }
+    }
+    
+    boolean isExt = ast.getChildCount() > 1;
+    descTableDesc descTblDesc = new descTableDesc(ctx.getResFile(), tableName, partSpec, isExt);
     rootTasks.add(TaskFactory.get(new DDLWork(descTblDesc), conf));
     LOG.info("analyzeDescribeTable done");
   }
   
+  private void analyzeShowPartitions(CommonTree ast) 
+  throws SemanticException {
+    showPartitionsDesc showPartsDesc;
+    String tableName = ast.getChild(0).getText();
+    showPartsDesc = new showPartitionsDesc(tableName, ctx.getResFile());
+    rootTasks.add(TaskFactory.get(new DDLWork(showPartsDesc), conf));
+  }
+  
   private void analyzeShowTables(CommonTree ast) 
   throws SemanticException {
     showTablesDesc showTblsDesc;
@@ -335,34 +383,32 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
     rootTasks.add(TaskFactory.get(new DDLWork(alterTblDesc), conf));
   }
 
-  private void analyzeAlterTableAddCols(CommonTree ast) 
+  private void analyzeAlterTableModifyCols(CommonTree ast, alterTableTypes alterType) 
   throws SemanticException {
     String tblName = ast.getChild(0).getText();
     List<FieldSchema> newCols = getColumns((CommonTree)ast.getChild(1));
-    Table tbl;
-    try {
-      tbl = db.getTable(tblName);
-    } catch (HiveException e) {
-      throw new SemanticException(e.getMessage()); 
-    }
-    List<FieldSchema> oldCols = tbl.getCols();
-    
-    // make sure the columns does not already exist
-    Iterator<FieldSchema> iterNewCols = newCols.iterator();
-    while (iterNewCols.hasNext()) {
-      FieldSchema newCol = iterNewCols.next();
-      String newColName  = newCol.getName();
-      Iterator<FieldSchema> iterOldCols = oldCols.iterator();
-      while (iterOldCols.hasNext()) {
-        String oldColName = iterOldCols.next().getName();
-        if (oldColName.equalsIgnoreCase(newColName)) 
-          throw new SemanticException(ErrorMsg.DUPLICATE_COLUMN_NAMES.getMsg());
-      }
-      oldCols.add(newCol);
-    }
-
-    alterTableDesc alterTblDesc = new alterTableDesc(tblName, oldCols);
+    alterTableDesc alterTblDesc = new alterTableDesc(tblName, newCols, alterType);
     rootTasks.add(TaskFactory.get(new DDLWork(alterTblDesc), conf));
   }
 
+  private void analyzeAlterTableDropParts(CommonTree ast) throws SemanticException {
+    String tblName = null;
+    List<HashMap<String, String>> partSpecs = new ArrayList<HashMap<String, String>>();
+    int childIndex = 0;
+    // get table metadata
+    tblName = ast.getChild(0).getText();
+    // get partition metadata if partition specified
+    for( childIndex = 1; childIndex < ast.getChildCount(); childIndex++) {
+      CommonTree partspec = (CommonTree) ast.getChild(childIndex);
+      HashMap<String, String> partSpec = new LinkedHashMap<String, String>();
+      for (int i = 0; i < partspec.getChildCount(); ++i) {
+        CommonTree partspec_val = (CommonTree) partspec.getChild(i);
+        String val = stripQuotes(partspec_val.getChild(1).getText());
+        partSpec.put(partspec_val.getChild(0).getText(), val);
+      }
+      partSpecs.add(partSpec);
+    }
+    dropTableDesc dropTblDesc = new dropTableDesc(tblName, partSpecs);
+    rootTasks.add(TaskFactory.get(new DDLWork(dropTblDesc), conf));
+  }
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java

@@ -28,6 +28,7 @@ public enum ErrorMsg {
   GENERIC_ERROR("Exception while processing"),
   INVALID_TABLE("Table not found"),
   INVALID_COLUMN("Invalid Column Reference"),
+  INVALID_PARTITION("Partition not found"),
   AMBIGOUS_COLUMN("Ambigous Column Reference"),
   AMBIGOUS_TABLE_ALIAS("Ambigous Table Alias"),
   INVALID_TABLE_ALIAS("Invalid Table Alias"),
@@ -47,12 +48,17 @@ public enum ErrorMsg {
   ILLEGAL_PATH("Path is not legal"),
   INVALID_NUMERICAL_CONSTANT("Invalid Numerical Constant"),
   INVALID_ARRAYINDEX_CONSTANT("Non Constant Expressions for Array Indexes not Supported"),
+  INVALID_MAPINDEX_CONSTANT("Non Constant Expression for Map Indexes not Supported"),
+  INVALID_MAPINDEX_TYPE("Map Key Type does not Match Index Expression Type"),
+  NON_COLLECTION_TYPE("[] not Valid on Non Collection Types"),
   SELECT_DISTINCT_WITH_GROUPBY("SELECT DISTINCT and GROUP BY can not be in the same query"),
   COLUMN_REPAEATED_IN_PARTITIONING_COLS("Column repeated in partitioning columns"),
   DUPLICATE_COLUMN_NAMES("Duplicate column names"),
   COLUMN_REPEATED_IN_CLUSTER_SORT("Same column cannot appear in cluster and sort by"),
   SAMPLE_RESTRICTION("Cannot Sample on More Than Two Columns"),
-  SAMPLE_COLUMN_NOT_FOUND("Sample Column Not Found");
+  SAMPLE_COLUMN_NOT_FOUND("Sample Column Not Found"),
+  NO_PARTITION_PREDICATE("No Partition Predicate Found"),
+  INVALID_DOT(". operator is only supported on struct or list of struct types");
   
   private String mesg;
   ErrorMsg(String mesg) {

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java

@@ -18,12 +18,15 @@
 
 package org.apache.hadoop.hive.ql.parse;
 
-import java.io.File;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.antlr.runtime.tree.CommonTree;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.TaskFactory;
 import org.apache.hadoop.hive.ql.plan.explainWork;
 
@@ -34,7 +37,7 @@ public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer {
     super(conf);
   }
 
-  public void analyze(CommonTree ast, Context ctx) throws SemanticException {
+  public void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException {
     
     // Create a semantic analyzer for the query
     BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, (CommonTree)ast.getChild(0));
@@ -46,9 +49,18 @@ public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer {
     }
     
     ctx.setResFile(new Path(getTmpFileName()));
-    
-    rootTasks.add(TaskFactory.get(new explainWork(ctx.getResFile(),
-                                                  sem.getRootTasks(),
+    List<Task<? extends Serializable>> tasks = sem.getRootTasks();
+    Task<? extends Serializable> fetchTask = sem.getFetchTask();
+    if (tasks == null) {
+    	if (fetchTask != null) {
+    		tasks = new ArrayList<Task<? extends Serializable>>();
+    		tasks.add(fetchTask);
+    	}
+    }
+    else if (fetchTask != null)
+    	tasks.add(fetchTask); 
+    		
+    rootTasks.add(TaskFactory.get(new explainWork(ctx.getResFile(), tasks,
                                                   ((CommonTree)ast.getChild(0)).toStringTree(),
                                                   extended), this.conf));
   }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java

@@ -36,7 +36,7 @@ public class FunctionSemanticAnalyzer extends BaseSemanticAnalyzer {
     super(conf);
   }
   
-  public void analyze(CommonTree ast, Context ctx) throws SemanticException {
+  public void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException {
     String functionName = ast.getChild(0).getText();
     String className = unescapeSQLString(ast.getChild(1).getText());
     createFunctionDesc desc = new createFunctionDesc(functionName, className);

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g

@@ -79,7 +79,10 @@ TOK_CREATETABLE;
 TOK_DESCTABLE;
 TOK_ALTERTABLE_RENAME;
 TOK_ALTERTABLE_ADDCOLS;
+TOK_ALTERTABLE_REPLACECOLS;
+TOK_ALTERTABLE_DROPPARTS;
 TOK_SHOWTABLES;
+TOK_SHOWPARTITIONS;
 TOK_CREATEEXTTABLE;
 TOK_DROPTABLE;
 TOK_TABCOLLIST;
@@ -102,6 +105,11 @@ TOK_TABSORTCOLNAMEDESC;
 TOK_CHARSETLITERAL;
 TOK_CREATEFUNCTION;
 TOK_EXPLAIN;
+TOK_TABLESERIALIZER;
+TOK_TABLSERDEPROPERTIES;
+TOK_TABLESERDEPROPLIST;
+TOK_LIMIT;
+TOKTABLESERDEPROPERTY;
 }
 
 
@@ -161,6 +169,7 @@ dropStatement
 alterStatement
     : alterStatementRename
     | alterStatementAddCol
+    | alterStatementDropPartitions
     ;
 
 alterStatementRename
@@ -169,16 +178,23 @@ alterStatementRename
     ;
 
 alterStatementAddCol
-    : KW_ALTER KW_TABLE Identifier KW_ADD KW_COLUMNS LPAREN columnNameTypeList RPAREN
-    -> ^(TOK_ALTERTABLE_ADDCOLS Identifier columnNameTypeList)
+    : KW_ALTER KW_TABLE Identifier (add=KW_ADD | replace=KW_REPLACE) KW_COLUMNS LPAREN columnNameTypeList RPAREN
+    -> {$add != null}? ^(TOK_ALTERTABLE_ADDCOLS Identifier columnNameTypeList)
+    ->                 ^(TOK_ALTERTABLE_REPLACECOLS Identifier columnNameTypeList)
+    ;
+
+alterStatementDropPartitions
+    : KW_ALTER KW_TABLE Identifier KW_DROP partitionSpec (COMMA partitionSpec)*
+    -> ^(TOK_ALTERTABLE_DROPPARTS Identifier partitionSpec+)
     ;
 
 descStatement
-    : KW_DESCRIBE Identifier  -> ^(TOK_DESCTABLE Identifier)
+    : KW_DESCRIBE (isExtended=KW_EXTENDED)? (tab=tabName)  -> ^(TOK_DESCTABLE $tab $isExtended?)
     ;
 
 showStatement
     : KW_SHOW KW_TABLES showStmtIdentifier?  -> ^(TOK_SHOWTABLES showStmtIdentifier?)
+    | KW_SHOW KW_PARTITIONS Identifier -> ^(TOK_SHOWPARTITIONS Identifier)
     ;
     
 createFunctionStatement
@@ -211,6 +227,23 @@ tableRowFormat
     :
       KW_ROW KW_FORMAT KW_DELIMITED tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier? 
     -> ^(TOK_TABLEROWFORMAT tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier?)
+    | KW_ROW KW_FORMAT KW_SERIALIZER name=StringLiteral tableSerializerProperties?
+    -> ^(TOK_TABLESERIALIZER $name tableSerializerProperties?)
+    ;
+
+tableSerializerProperties
+    :
+      KW_WITH KW_PROPERTIES LPAREN propertiesList RPAREN -> ^(TOK_TABLSERDEPROPERTIES propertiesList)
+    ;
+
+propertiesList
+    :
+      keyValueProperty (COMMA keyValueProperty)* -> ^(TOK_TABLESERDEPROPLIST keyValueProperty+)
+    ;
+
+keyValueProperty
+    :
+      key=StringLiteral EQUAL value=StringLiteral -> ^(TOKTABLESERDEPROPERTY $key $value)
     ;
 
 tableRowFormatFieldIdentifier
@@ -328,14 +361,16 @@ regular_body
    whereClause?
    groupByClause?
    orderByClause?
-   clusterByClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT insertClause selectClause whereClause? groupByClause? orderByClause? clusterByClause?))
+   clusterByClause? 
+   limitClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT insertClause selectClause whereClause? groupByClause? orderByClause? clusterByClause? limitClause?))
    |
    selectClause
    fromClause
    whereClause?
    groupByClause?
    orderByClause?
-   clusterByClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) selectClause whereClause? groupByClause? orderByClause? clusterByClause?))
+   clusterByClause? 
+   limitClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) selectClause whereClause? groupByClause? orderByClause? clusterByClause? limitClause?))
    ;
 
 
@@ -346,13 +381,15 @@ body
    whereClause?
    groupByClause?
    orderByClause? 
-   clusterByClause? -> ^(TOK_INSERT insertClause? selectClause whereClause? groupByClause? orderByClause? clusterByClause?)
+   clusterByClause? 
+   limitClause? -> ^(TOK_INSERT insertClause? selectClause whereClause? groupByClause? orderByClause? clusterByClause? limitClause?)
    |
    selectClause
    whereClause?
    groupByClause?
    orderByClause? 
-   clusterByClause? -> ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) selectClause whereClause? groupByClause? orderByClause? clusterByClause?)
+   clusterByClause? 
+   limitClause? -> ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) selectClause whereClause? groupByClause? orderByClause? clusterByClause? limitClause?)
    ;
    
 insertClause
@@ -367,6 +404,11 @@ destination
    | KW_TABLE tabName -> ^(tabName)
    ;
 
+limitClause
+   :
+   KW_LIMIT num=Number -> ^(TOK_LIMIT $num)
+   ;
+
 //----------------------- Rules for parsing selectClause -----------------------------
 // select a,b,c ...
 selectClause
@@ -581,7 +623,7 @@ precedenceFieldExpression
 
 precedenceUnaryOperator
     :
-    MINUS | TILDE
+    PLUS | MINUS | TILDE
     ;
 
 precedenceUnaryExpression
@@ -741,6 +783,7 @@ KW_RIGHT : 'RIGHT';
 KW_FULL : 'FULL';
 KW_ON : 'ON';
 KW_PARTITION : 'PARTITION';
+KW_PARTITIONS : 'PARTITIONS';
 KW_TABLE: 'TABLE';
 KW_TABLES: 'TABLES';
 KW_SHOW: 'SHOW';
@@ -798,6 +841,7 @@ KW_OUT: 'OUT';
 KW_OF: 'OF';
 KW_CAST: 'CAST';
 KW_ADD: 'ADD';
+KW_REPLACE: 'REPLACE';
 KW_COLUMNS: 'COLUMNS';
 KW_RLIKE: 'RLIKE';
 KW_REGEXP: 'REGEXP';
@@ -805,6 +849,10 @@ KW_TEMPORARY: 'TEMPORARY';
 KW_FUNCTION: 'FUNCTION';
 KW_EXPLAIN: 'EXPLAIN';
 KW_EXTENDED: 'EXTENDED';
+KW_SERIALIZER: 'SERIALIZER';
+KW_WITH: 'WITH';
+KW_PROPERTIES: 'SERDEPROPERTIES';
+KW_LIMIT: 'LIMIT';
 
 // Operators
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java

@@ -156,7 +156,7 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
   }
 
   @Override
-  public void analyze(CommonTree ast, Context ctx) throws SemanticException {
+  public void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException {
     isLocal = isOverWrite = false;
     Tree from_t = ast.getChild(0);
     Tree table_t = ast.getChild(1);
@@ -185,7 +185,7 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
     }
 
     // initialize destination table/partition
-    tableSpec ts = new tableSpec(db, (CommonTree) table_t);
+    tableSpec ts = new tableSpec(db, (CommonTree) table_t, true);
     URI toURI = (ts.partHandle != null) ? ts.partHandle.getDataLocation() : ts.tableHandle.getDataLocation();
 
     // make sure the arguments make sense

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java

@@ -17,14 +17,6 @@
  */
 
 package org.apache.hadoop.hive.ql.parse;
-/*
- * PartitionPruner.java
- *
- * Created on April 9, 2008, 3:48 PM
- *
- * To change this template, choose Tools | Template Manager
- * and open the template in the editor.
- */
 
 import java.util.*;
 
@@ -73,7 +65,7 @@ public class PartitionPruner {
     this.tableAlias = tableAlias;
     this.metaData = metaData;
     this.tab = metaData.getTableForAlias(tableAlias);
-    this.prunerExpr = new exprNodeConstantDesc(Boolean.TRUE);
+    this.prunerExpr = null;
   }
 
   /**
@@ -106,8 +98,8 @@ public class PartitionPruner {
       case HiveParser.TOK_COLREF: {
 
         assert(expr.getChildCount() == 2);
-        String tabAlias = SemanticAnalyzer.getTableName(expr);
-        String colName = SemanticAnalyzer.getSerDeFieldExpression(expr);
+        String tabAlias = expr.getChild(0).getText();
+        String colName = expr.getChild(1).getText();
         if (tabAlias == null || colName == null) {
           throw new SemanticException(ErrorMsg.INVALID_XPATH.getMsg(expr));
         }
@@ -115,11 +107,17 @@ public class PartitionPruner {
         if (tabAlias.equals(tableAlias) && tab.isPartitionKey(colName)) {
           desc = new exprNodeColumnDesc(String.class, colName); 
         } else {
-          // might be a column from another table
           try {
-            TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(
-                this.metaData.getTableForAlias(tabAlias).getDeserializer().getObjectInspector());
-            desc = new exprNodeConstantDesc(typeInfo.getStructFieldTypeInfo(colName), null);
+            // might be a column from another table
+            Table t = this.metaData.getTableForAlias(tabAlias);
+            if (t.isPartitionKey(colName)) {
+              desc = new exprNodeConstantDesc(String.class, null);
+            }
+            else {
+              TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(
+                                                                               this.metaData.getTableForAlias(tabAlias).getDeserializer().getObjectInspector());
+              desc = new exprNodeConstantDesc(typeInfo.getStructFieldTypeInfo(colName), null);
+            }
           } catch (SerDeException e){
             throw new RuntimeException(e);
           }
@@ -189,6 +187,37 @@ public class PartitionPruner {
     return false;
   }
   
+  public boolean hasPartitionPredicate(CommonTree expr) {
+
+    int tokType = expr.getType();
+    boolean hasPPred = false;
+    switch (tokType) {
+      case HiveParser.TOK_COLREF: {
+
+        assert(expr.getChildCount() == 2);
+        String tabAlias = expr.getChild(0).getText();
+        String colName = expr.getChild(1).getText();
+        if (tabAlias.equals(tableAlias) && tab.isPartitionKey(colName)) {
+          hasPPred = true;
+        }
+        break;
+      }
+
+      default: {
+        boolean isFunction = (expr.getType() == HiveParser.TOK_FUNCTION);
+        
+        // Create all children
+        int childrenBegin = (isFunction ? 1 : 0);
+        for (int ci=childrenBegin; ci<expr.getChildCount(); ci++) {
+          hasPPred = (hasPPred || hasPartitionPredicate((CommonTree)expr.getChild(ci)));
+        }
+        break;
+      }
+    }
+
+    return hasPPred;
+  }
+
   /** Add an expression */
   @SuppressWarnings("nls")
   public void addExpression(CommonTree expr) throws SemanticException {
@@ -197,7 +226,10 @@ public class PartitionPruner {
     // Ignore null constant expressions
     if (!(desc instanceof exprNodeConstantDesc) || ((exprNodeConstantDesc)desc).getValue() != null ) {
       LOG.trace("adding pruning expr = " + desc);
-      this.prunerExpr = SemanticAnalyzer.getFuncExprNodeDesc("AND", this.prunerExpr, desc);
+      if (this.prunerExpr == null)
+      	this.prunerExpr = desc;
+      else
+        this.prunerExpr = SemanticAnalyzer.getFuncExprNodeDesc("OR", this.prunerExpr, desc);
     }
   }
   
@@ -208,13 +240,15 @@ public class PartitionPruner {
     LOG.trace("tabname = " + this.tab.getName());
     LOG.trace("prune Expression = " + this.prunerExpr);
 
-    HashSet<Partition> ret_parts = new HashSet<Partition>();
+    LinkedHashSet<Partition> ret_parts = new LinkedHashSet<Partition>();
     try {
       StructObjectInspector rowObjectInspector = (StructObjectInspector)this.tab.getDeserializer().getObjectInspector();
       Object[] rowWithPart = new Object[2];
       InspectableObject inspectableObject = new InspectableObject();
-      
-      ExprNodeEvaluator evaluator = ExprNodeEvaluatorFactory.get(this.prunerExpr);
+     
+      ExprNodeEvaluator evaluator = null;
+      if (this.prunerExpr != null)
+        evaluator = ExprNodeEvaluatorFactory.get(this.prunerExpr);
       for(Partition part: Hive.get().getPartitions(this.tab)) {
         // Set all the variables here
         LinkedHashMap<String, String> partSpec = part.getSpec();
@@ -237,14 +271,18 @@ public class PartitionPruner {
         StructObjectInspector rowWithPartObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(ois);
         
         // evaluate the expression tree
-        evaluator.evaluate(rowWithPart, rowWithPartObjectInspector, inspectableObject);
-        LOG.trace("prune result for partition " + partSpec + ": " + inspectableObject.o);
-        if (!Boolean.FALSE.equals(inspectableObject.o)) {
-          LOG.debug("retained partition: " + partSpec);
-          ret_parts.add(part);
-        } else {
-          LOG.trace("pruned partition: " + partSpec);
+        if (evaluator != null) {
+          evaluator.evaluate(rowWithPart, rowWithPartObjectInspector, inspectableObject);
+          LOG.trace("prune result for partition " + partSpec + ": " + inspectableObject.o);
+          if (!Boolean.FALSE.equals(inspectableObject.o)) {
+            LOG.debug("retained partition: " + partSpec);
+            ret_parts.add(part);
+          } else {
+            LOG.trace("pruned partition: " + partSpec);
+          }
         }
+        else
+        	ret_parts.add(part);
       }
     }
     catch (Exception e) {

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java

@@ -22,6 +22,7 @@ import java.util.*;
 
 import org.apache.hadoop.hive.ql.parse.QBParseInfo;
 import org.apache.hadoop.hive.ql.parse.QBMetaData;
+import org.apache.hadoop.hive.ql.metadata.Table;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -45,6 +46,7 @@ public class QB {
   private QBMetaData qbm;
   private QBJoinTree qbjoin;
   private String id;
+  private boolean isQuery;
 
   public void print(String msg) {
     LOG.info(msg + "alias=" + qbp.getAlias());
@@ -59,6 +61,9 @@ public class QB {
   public QB(String outer_id, String alias, boolean isSubQ) {
     aliasToTabs = new HashMap<String, String>();
     aliasToSubq = new HashMap<String, QBExpr>();
+    if (alias != null) {
+      alias = alias.toLowerCase();
+    }
     qbp = new QBParseInfo(alias, isSubQ);
     qbm = new QBMetaData();
     this.id = (outer_id == null ? alias : outer_id + ":" + alias);
@@ -85,6 +90,7 @@ public class QB {
   }
 
   public boolean exists(String alias) {
+    alias = alias.toLowerCase();
     if (aliasToTabs.get(alias) != null || aliasToSubq.get(alias) != null)
       return true;
 
@@ -92,11 +98,11 @@ public class QB {
   }
 
   public void setTabAlias(String alias, String tabName) {
-    aliasToTabs.put(alias, tabName);
+    aliasToTabs.put(alias.toLowerCase(), tabName);
   }
 
   public void setSubqAlias(String alias, QBExpr qbexpr) {
-    aliasToSubq.put(alias, qbexpr);
+    aliasToSubq.put(alias.toLowerCase(), qbexpr);
   }
 
   public String getId() {
@@ -128,11 +134,11 @@ public class QB {
   }
 
   public QBExpr getSubqForAlias(String alias) {
-    return aliasToSubq.get(alias);
+    return aliasToSubq.get(alias.toLowerCase());
   }
 
   public String getTabNameForAlias(String alias) {
-    return aliasToTabs.get(alias);
+    return aliasToTabs.get(alias.toLowerCase());
   }
 
   public QBJoinTree getQbJoinTree() {
@@ -142,4 +148,24 @@ public class QB {
   public void setQbJoinTree(QBJoinTree qbjoin) {
     this.qbjoin = qbjoin;
   }
+
+  public void setIsQuery(boolean isQuery) {
+    this.isQuery = isQuery;
+  }
+
+  public boolean getIsQuery() {
+    return isQuery;
+  }
+
+  public boolean isSelectStarQuery() {
+    if (!qbp.isSelectStarQuery() || !aliasToSubq.isEmpty())
+      return false;
+
+    Iterator<Map.Entry<String, Table>> iter = qbm.getAliasToTable().entrySet().iterator();
+    Table tab = ((Map.Entry<String, Table>)iter.next()).getValue();
+    if (tab.isPartitioned())
+      return false;
+    
+    return true;
+  }
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/QBMetaData.java

@@ -64,8 +64,16 @@ public class QBMetaData {
     this.outTypes.add(cls);
   }
 
+  // All getXXX needs toLowerCase() because they are directly called from SemanticAnalyzer
+  // All setXXX does not need it because they are called from QB which already lowercases
+  // the aliases.
+
+  public HashMap<String, Table> getAliasToTable() {
+    return aliasToTable;
+  }
+
   public Table getTableForAlias(String alias) {
-    return this.aliasToTable.get(alias);
+    return this.aliasToTable.get(alias.toLowerCase());
   }
 
   public void setSrcForAlias(String alias, Table tab) {
@@ -89,23 +97,23 @@ public class QBMetaData {
   }
 
   public Integer getDestTypeForAlias(String alias) {
-    return this.nameToDestType.get(alias);
+    return this.nameToDestType.get(alias.toLowerCase());
   }
 
   public Table getDestTableForAlias(String alias) {
-    return this.nameToDestTable.get(alias);
+    return this.nameToDestTable.get(alias.toLowerCase());
   }
 
   public Partition getDestPartitionForAlias(String alias) {
-    return this.nameToDestPartition.get(alias);
+    return this.nameToDestPartition.get(alias.toLowerCase());
   }
 
   public String getDestFileForAlias(String alias) {
-    return this.nameToDestFile.get(alias);
+    return this.nameToDestFile.get(alias.toLowerCase());
   }
 
   public Table getSrcForAlias(String alias) {
-    return this.aliasToTable.get(alias);
+    return this.aliasToTable.get(alias.toLowerCase());
   }
   
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java

@@ -32,6 +32,7 @@ import org.apache.commons.logging.LogFactory;
 public class QBParseInfo {
 
   private boolean isSubQ;
+  private boolean canOptTopQ;
   private String alias;
   private CommonTree joinExpr;
   private HashMap<String, CommonTree> aliasToSrc;
@@ -41,6 +42,8 @@ public class QBParseInfo {
   private HashMap<String, CommonTree> destToWhereExpr;
   private HashMap<String, CommonTree> destToGroupby;
   private HashMap<String, CommonTree> destToClusterby;
+  private HashMap<String, Integer>    destToLimit;
+  private int outerQueryLimit;
 
   // used by GroupBy
   private HashMap<String, HashMap<String, CommonTree> > destToAggregationExprs;
@@ -57,12 +60,15 @@ public class QBParseInfo {
     this.destToWhereExpr = new HashMap<String, CommonTree>();
     this.destToGroupby = new HashMap<String, CommonTree>();
     this.destToClusterby = new HashMap<String, CommonTree>();
+    this.destToLimit = new HashMap<String, Integer>();
     
     this.destToAggregationExprs = new HashMap<String, HashMap<String, CommonTree> >();
     this.destToDistinctFuncExpr = new HashMap<String, CommonTree>();
     
     this.alias = alias;
     this.isSubQ = isSubQ;
+    this.canOptTopQ = false;
+    this.outerQueryLimit = -1;
   }
 
   public void setAggregationExprsForClause(String clause, HashMap<String, CommonTree> aggregationTrees) {
@@ -102,7 +108,7 @@ public class QBParseInfo {
   }
 
   public void setSrcForAlias(String alias, CommonTree ast) {
-    this.aliasToSrc.put(alias, ast);
+    this.aliasToSrc.put(alias.toLowerCase(), ast);
   }
 
   public Set<String> getClauseNames() {
@@ -134,7 +140,7 @@ public class QBParseInfo {
   }
 
   public CommonTree getSrcForAlias(String alias) {
-    return this.aliasToSrc.get(alias);
+    return this.aliasToSrc.get(alias.toLowerCase());
   }
 
   public String getAlias() {
@@ -145,6 +151,14 @@ public class QBParseInfo {
     return this.isSubQ;
   }
 
+  public boolean getCanOptTopQ() {
+    return this.canOptTopQ;
+  }
+
+  public void setCanOptTopQ(boolean canOptTopQ) {
+    this.canOptTopQ = canOptTopQ;
+  }
+  
   public CommonTree getJoinExpr() {
     return this.joinExpr;
   }
@@ -152,12 +166,87 @@ public class QBParseInfo {
   public void setJoinExpr(CommonTree joinExpr) {
     this.joinExpr = joinExpr;
   }
-  
+
   public TableSample getTabSample(String alias) {
-    return this.nameToSample.get(alias);
+    return this.nameToSample.get(alias.toLowerCase());
   }
   
   public void setTabSample(String alias, TableSample tableSample) {
-    this.nameToSample.put(alias, tableSample);
+    this.nameToSample.put(alias.toLowerCase(), tableSample);
+  }
+
+  public void setDestLimit(String dest, Integer limit) {
+    this.destToLimit.put(dest, limit);
   }
+
+  public Integer getDestLimit(String dest) {
+    return this.destToLimit.get(dest);
+  }
+
+	/**
+	 * @return the outerQueryLimit
+	 */
+	public int getOuterQueryLimit() {
+		return outerQueryLimit;
+	}
+
+	/**
+	 * @param outerQueryLimit the outerQueryLimit to set
+	 */
+	public void setOuterQueryLimit(int outerQueryLimit) {
+		this.outerQueryLimit = outerQueryLimit;
+	}
+
+  public boolean isSelectStarQuery() {
+    if (isSubQ || 
+       (joinExpr != null) ||
+       (!nameToSample.isEmpty()) ||
+       (!destToWhereExpr.isEmpty()) ||
+       (!destToGroupby.isEmpty()) ||
+       (!destToClusterby.isEmpty()))
+      return false;
+    
+    Iterator<Map.Entry<String, HashMap<String, CommonTree>>> aggrIter = destToAggregationExprs.entrySet().iterator();
+    while (aggrIter.hasNext()) {
+      HashMap<String, CommonTree> h = aggrIter.next().getValue();
+      if ((h != null) && (!h.isEmpty()))
+        return false;
+    }
+      	
+    if (!destToDistinctFuncExpr.isEmpty()) {
+      Iterator<Map.Entry<String, CommonTree>> distn = destToDistinctFuncExpr.entrySet().iterator();
+      while (distn.hasNext()) {
+        CommonTree ct = distn.next().getValue();
+        if (ct != null) 
+          return false;
+      }
+    }
+        
+    Iterator<Map.Entry<String, CommonTree>> iter = nameToDest.entrySet().iterator();
+    while (iter.hasNext()) {
+      Map.Entry<String, CommonTree> entry = iter.next();
+      CommonTree v = entry.getValue();
+      if (!(((CommonTree)v.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE))
+        return false;
+    }
+      	
+    iter = destToSelExpr.entrySet().iterator();
+    while (iter.hasNext()) {
+      Map.Entry<String, CommonTree> entry = iter.next();
+      CommonTree selExprList = entry.getValue();
+      // Iterate over the selects
+      for (int i = 0; i < selExprList.getChildCount(); ++i) {
+        
+        // list of the columns
+        CommonTree selExpr = (CommonTree) selExprList.getChild(i);
+        CommonTree sel = (CommonTree)selExpr.getChild(0);
+        
+        if (sel.getToken().getType() != HiveParser.TOK_ALLCOLREF)
+          return false;
+      }
+    }
+
+    return true;
+  }
+  
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java

@@ -50,6 +50,9 @@ public class RowResolver {
   }
   
   public void put(String tab_alias, String col_alias, ColumnInfo colInfo) {
+    if (tab_alias != null) {
+      tab_alias = tab_alias.toLowerCase();
+    }
     col_alias = col_alias.toLowerCase();
     if (rowSchema.getSignature() == null) {
       rowSchema.setSignature(new Vector<ColumnInfo>());
@@ -71,10 +74,11 @@ public class RowResolver {
   }
 
   public boolean hasTableAlias(String tab_alias) {
-    return rslvMap.get(tab_alias) != null ? true : false;
+    return rslvMap.get(tab_alias.toLowerCase()) != null;
   }
 
   public ColumnInfo get(String tab_alias, String col_alias) {
+    tab_alias = tab_alias.toLowerCase();
     col_alias = col_alias.toLowerCase();
     HashMap<String, ColumnInfo> f_map = rslvMap.get(tab_alias);
     if (f_map == null) {
@@ -88,7 +92,7 @@ public class RowResolver {
   }
  
   public HashMap<String, ColumnInfo> getFieldMap(String tab_alias) {
-    return rslvMap.get(tab_alias);
+    return rslvMap.get(tab_alias.toLowerCase());
   }
 
   public int getPosition(String internalName) {

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java

@@ -36,8 +36,11 @@ public class SemanticAnalyzerFactory {
       case HiveParser.TOK_DROPTABLE: 
       case HiveParser.TOK_DESCTABLE: 
       case HiveParser.TOK_ALTERTABLE_ADDCOLS:
+      case HiveParser.TOK_ALTERTABLE_REPLACECOLS:
       case HiveParser.TOK_ALTERTABLE_RENAME:
-      case HiveParser.TOK_SHOWTABLES: 
+      case HiveParser.TOK_ALTERTABLE_DROPPARTS:
+      case HiveParser.TOK_SHOWTABLES:
+      case HiveParser.TOK_SHOWPARTITIONS:
         return new DDLSemanticAnalyzer(conf);
       case HiveParser.TOK_CREATEFUNCTION: 
         return new FunctionSemanticAnalyzer(conf);

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java

@@ -22,11 +22,12 @@ import java.io.Serializable;
 
 public class DDLWork implements Serializable {
   private static final long serialVersionUID = 1L;
-  private createTableDesc  createTblDesc;
-  private dropTableDesc    dropTblDesc;
-  private alterTableDesc   alterTblDesc;
-  private showTablesDesc   showTblsDesc;
-  private descTableDesc    descTblDesc;
+  private createTableDesc      createTblDesc;
+  private dropTableDesc        dropTblDesc;
+  private alterTableDesc       alterTblDesc;
+  private showTablesDesc       showTblsDesc;
+  private showPartitionsDesc   showPartsDesc;
+  private descTableDesc        descTblDesc;
 
   public DDLWork() { }
 
@@ -65,6 +66,13 @@ public class DDLWork implements Serializable {
     this.showTblsDesc = showTblsDesc;
   }
 
+  /**
+   * @param showPartsDesc
+   */
+  public DDLWork(showPartitionsDesc showPartsDesc) {
+    this.showPartsDesc = showPartsDesc;
+  }
+
   /**
    * @return the createTblDesc
    */
@@ -125,6 +133,22 @@ public class DDLWork implements Serializable {
     this.showTblsDesc = showTblsDesc;
   }
 
+
+  /**
+   * @return the showPartsDesc
+   */
+  @explain(displayName="Show Partitions Operator")
+  public showPartitionsDesc getShowPartsDesc() {
+    return showPartsDesc;
+  }
+
+  /**
+   * @param showPartsDesc the showPartsDesc to set
+   */
+  public void setShowPartsDesc(showPartitionsDesc showPartsDesc) {
+    this.showPartsDesc = showPartsDesc;
+  }
+
   /**
    * @return the descTblDesc
    */

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java

@@ -63,21 +63,14 @@ public class PlanUtils {
 
   
   // We will make reduce key and reduce value TableDesc with configurable SerDes   
-  public static reduceSinkDesc getReduceSinkDesc(ArrayList<exprNodeDesc> keyCols,
-      ArrayList<exprNodeDesc> valueCols, int numPartitionFields) {
+  public static reduceSinkDesc getReduceSinkDesc(ArrayList<exprNodeDesc> keyCols, 
+                                                 ArrayList<exprNodeDesc> valueCols, 
+                                                 int tag, int numPartitionFields, 
+                                                 int numReducers, boolean inferNumReducers) {
      
-     return new reduceSinkDesc(keyCols, valueCols, numPartitionFields,
-         getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(keyCols.size())),
-         getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(valueCols.size())));
-  }
-  
-  // We will make reduce key and reduce value TableDesc with configurable SerDes   
-  public static reduceSinkDesc getReduceSinkDesc(ArrayList<exprNodeDesc> keyCols,
-      ArrayList<exprNodeDesc> valueCols, int tag, int numPartitionFields) {
-     
-     return new reduceSinkDesc(keyCols, valueCols, tag, numPartitionFields,
-         getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(keyCols.size())),
-         getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(valueCols.size())));
+    return new reduceSinkDesc(keyCols, valueCols, tag, numPartitionFields, numReducers, inferNumReducers,
+      getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(keyCols.size())),
+      getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(valueCols.size())));
   }
 
   // We should read the TableDesc from gWork when it is available.   

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/alterTableDesc.java

@@ -19,7 +19,6 @@
 package org.apache.hadoop.hive.ql.plan;
 
 import java.io.Serializable;
-import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -29,7 +28,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities;
 public class alterTableDesc extends ddlDesc implements Serializable 
 {
   private static final long serialVersionUID = 1L;
-  public static enum alterTableTypes {RENAME, ADDCOLS};
+  public static enum alterTableTypes {RENAME, ADDCOLS, REPLACECOLS};
     
   alterTableTypes      op;
   String               oldName;
@@ -50,8 +49,8 @@ public class alterTableDesc extends ddlDesc implements Serializable
    * @param name name of the table
    * @param newCols new columns to be added
    */
-  public alterTableDesc(String name, List<FieldSchema> newCols) {
-    op = alterTableTypes.ADDCOLS;
+  public alterTableDesc(String name, List<FieldSchema> newCols, alterTableTypes alterType) {
+    this.op = alterType;
     this.oldName = name;
     this.newCols = newCols;
   }
@@ -100,6 +99,8 @@ public class alterTableDesc extends ddlDesc implements Serializable
       return "rename";
     case ADDCOLS:
       return "add columns";
+    case REPLACECOLS:
+      return "replace columns";
     }
     
     return "unknown";
@@ -128,4 +129,5 @@ public class alterTableDesc extends ddlDesc implements Serializable
   public void setNewCols(List<FieldSchema> newCols) {
     this.newCols = newCols;
   }
+
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.plan;
 
 import java.io.Serializable;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Order;
@@ -43,6 +44,8 @@ public class createTableDesc extends ddlDesc implements Serializable
   String              comment;
   boolean             isSequenceFile;
   String              location;
+  String              serName;
+  Map<String, String> mapProp;
   
   public createTableDesc(String tableName, boolean isExternal, 
                          List<FieldSchema> cols, List<FieldSchema> partCols,
@@ -51,7 +54,7 @@ public class createTableDesc extends ddlDesc implements Serializable
                          String collItemDelim,
                          String mapKeyDelim, String lineDelim, 
                          String comment, boolean isSequenceFile, 
-                         String location) {
+                         String location, String serName, Map<String, String> mapProp) {
     this.tableName       = tableName;
     this.isExternal      = isExternal;
     this.bucketCols      = bucketCols;
@@ -66,6 +69,8 @@ public class createTableDesc extends ddlDesc implements Serializable
     this.mapKeyDelim     = mapKeyDelim;
     this.numBuckets      = numBuckets;
     this.partCols        = partCols;
+    this.serName         = serName;
+    this.mapProp         = mapProp;
   }
 
   @explain(displayName="name")
@@ -166,7 +171,7 @@ public class createTableDesc extends ddlDesc implements Serializable
     this.comment = comment;
   }
 
-  @explain(displayName="isSequenceFile")  
+  @explain(displayName="isSequenceFile")
   public boolean isSequenceFile() {
     return isSequenceFile;
   }
@@ -207,4 +212,35 @@ public class createTableDesc extends ddlDesc implements Serializable
   public void setSortCols(List<Order> sortCols) {
     this.sortCols = sortCols;
   }
+
+	/**
+	 * @return the serDeName
+	 */
+  @explain(displayName="serde name")
+	public String getSerName() {
+		return serName;
+	}
+
+	/**
+	 * @param serName the serName to set
+	 */
+	public void setSerName(String serName) {
+		this.serName = serName;
+	}
+
+	/**
+	 * @return the serDe properties
+	 */
+  @explain(displayName="serde properties")
+  public Map<String,String> getMapProp() {
+		return mapProp;
+	}
+
+	/**
+	 * @param mapProp the map properties to set
+	 */
+	public void setMapProp(Map<String,String> mapProp) {
+		this.mapProp = mapProp;
+	}
+  
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/descTableDesc.java

@@ -19,6 +19,8 @@
 package org.apache.hadoop.hive.ql.plan;
 
 import java.io.Serializable;
+import java.util.HashMap;
+
 import org.apache.hadoop.fs.Path;
 
 @explain(displayName="Describe Table")
@@ -26,27 +28,68 @@ public class descTableDesc extends ddlDesc implements Serializable
 {
   private static final long serialVersionUID = 1L;
   
-  String            tableName;
+  String tableName;
+  HashMap<String, String> partSpec;
   Path              resFile;
+  boolean           isExt;
   
   /**
+   * @param isExt
+   * @param partSpec
    * @param resFile
    * @param tableName
    */
-  public descTableDesc(Path resFile, String tableName) {
+  public descTableDesc(Path resFile, String tableName, HashMap<String, String> partSpec, boolean isExt) {
+    this.isExt = isExt;
+    this.partSpec = partSpec;
     this.resFile = resFile;
     this.tableName = tableName;
   }
 
+  /**
+   * @return the isExt
+   */
+  public boolean isExt() {
+    return isExt;
+  }
+
+  /**
+   * @param isExt the isExt to set
+   */
+  public void setExt(boolean isExt) {
+    this.isExt = isExt;
+  }
+
+  /**
+   * @return the tableName
+   */
   @explain(displayName="table")
   public String getTableName() {
     return tableName;
   }
 
+  /**
+   * @param tableName the tableName to set
+   */
   public void setTableName(String tableName) {
     this.tableName = tableName;
   }
 
+  /**
+   * @return the partSpec
+   */
+  @explain(displayName="partition")
+  public HashMap<String, String> getPartSpec() {
+    return partSpec;
+  }
+
+  /**
+   * @param partSpec the partSpec to set
+   */
+  public void setPartSpecs(HashMap<String, String> partSpec) {
+    this.partSpec = partSpec;
+  }
+
   /**
    * @return the resFile
    */

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/dropTableDesc.java

@@ -19,6 +19,8 @@
 package org.apache.hadoop.hive.ql.plan;
 
 import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
 
 @explain(displayName="Drop Table")
 public class dropTableDesc extends ddlDesc implements Serializable 
@@ -26,12 +28,19 @@ public class dropTableDesc extends ddlDesc implements Serializable
   private static final long serialVersionUID = 1L;
   
   String            tableName;
+  List<HashMap<String, String>> partSpecs;
 
   /**
    * @param tableName
    */
   public dropTableDesc(String tableName) {
     this.tableName = tableName;
+    this.partSpecs = null;
+  }
+
+  public dropTableDesc(String tableName, List<HashMap<String, String>> partSpecs) {
+    this.tableName = tableName;
+    this.partSpecs = partSpecs;
   }
 
   /**
@@ -48,4 +57,18 @@ public class dropTableDesc extends ddlDesc implements Serializable
   public void setTableName(String tableName) {
     this.tableName = tableName;
   }
+
+  /**
+   * @return the partSpecs
+   */
+  public List<HashMap<String, String>> getPartSpecs() {
+    return partSpecs;
+  }
+
+  /**
+   * @param partSpecs the partSpecs to set
+   */
+  public void setPartSpecs(List<HashMap<String, String>> partSpecs) {
+    this.partSpecs = partSpecs;
+  }
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java

@@ -26,11 +26,13 @@ import org.apache.hadoop.hive.ql.typeinfo.TypeInfoFactory;
 public class exprNodeColumnDesc extends exprNodeDesc implements Serializable {
   private static final long serialVersionUID = 1L;
   private String column;
+  private boolean isVirtual;
   
   public exprNodeColumnDesc() {}
   public exprNodeColumnDesc(TypeInfo typeInfo, String column) {
     super(typeInfo);
     this.column = column;
+    this.isVirtual = isVirtual;
   }
   public exprNodeColumnDesc(Class<?> c, String column) {
     super(TypeInfoFactory.getPrimitiveTypeInfo(c));
@@ -42,6 +44,7 @@ public class exprNodeColumnDesc extends exprNodeDesc implements Serializable {
   public void setColumn(String column) {
     this.column = column;
   }
+
   public String toString() {
     return "Column[" + column + "]";
   }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFieldDesc.java

@@ -25,13 +25,18 @@ import org.apache.hadoop.hive.ql.typeinfo.TypeInfo;
 public class exprNodeFieldDesc extends exprNodeDesc implements Serializable {
   private static final long serialVersionUID = 1L;
   exprNodeDesc desc;
-  String fieldName;
+  String fieldName;
+  
+  // Used to support a.b where a is a list of struct that contains a field called b.
+  // a.b will return an array that contains field b of all elements of array a. 
+  Boolean isList;
   
   public exprNodeFieldDesc() {}
-  public exprNodeFieldDesc(TypeInfo typeInfo, exprNodeDesc desc, String fieldName) {
+  public exprNodeFieldDesc(TypeInfo typeInfo, exprNodeDesc desc, String fieldName, Boolean isList) {
     super(typeInfo);
     this.desc = desc;
-    this.fieldName = fieldName;    
+    this.fieldName = fieldName;
+    this.isList = isList;
   }
   
   public exprNodeDesc getDesc() {
@@ -45,7 +50,14 @@ public class exprNodeFieldDesc extends exprNodeDesc implements Serializable {
   }
   public void setFieldName(String fieldName) {
     this.fieldName = fieldName;
-  }
+  }
+  public Boolean getIsList() {
+    return isList;
+  }
+  public void setIsList(Boolean isList) {
+    this.isList = isList;
+  }
+  
   @Override
   public String toString() {
     return this.desc.toString() + "." + this.fieldName;

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/fetchWork.java

@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.plan;
+
+import java.io.Serializable;
+import java.util.Properties;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.mapred.InputFormat;
+
+@explain(displayName="Fetch Operator")
+public class fetchWork implements Serializable {
+  private static final long serialVersionUID = 1L;
+
+  //  private loadFileDesc loadFileWork;
+  //  private tableDesc    tblDesc;
+  private Path srcDir;
+  private Properties schema;
+  private Class<? extends Deserializer> deserializerClass;
+  private Class<? extends InputFormat> inputFormatClass;
+  private int limit;
+
+  public fetchWork() { }
+
+	/**
+	 * @param deserializer
+	 * @param deserializerClass
+	 * @param inputFormatClass
+	 * @param schema
+	 * @param srcDir
+	 */
+	public fetchWork(Path srcDir,
+			Class<? extends Deserializer> deserializerClass,
+			Class<? extends InputFormat> inputFormatClass, Properties schema,
+			int limit) {
+		this.srcDir = srcDir;
+		this.deserializerClass = deserializerClass;
+		this.inputFormatClass = inputFormatClass;
+		this.schema = schema;
+		this.limit = limit;
+	}
+
+	/**
+	 * @return the srcDir
+	 */
+  @explain(displayName="source")
+	public Path getSrcDir() {
+		return srcDir;
+	}
+
+	/**
+	 * @param srcDir the srcDir to set
+	 */
+	public void setSrcDir(Path srcDir) {
+		this.srcDir = srcDir;
+	}
+
+	/**
+	 * @return the schema
+	 */
+	public Properties getSchema() {
+		return schema;
+	}
+
+	/**
+	 * @param schema the schema to set
+	 */
+	public void setSchema(Properties schema) {
+		this.schema = schema;
+	}
+
+	/**
+	 * @return the deserializerClass
+	 */
+	public Class<? extends Deserializer> getDeserializerClass() {
+		return deserializerClass;
+	}
+
+	/**
+	 * @param deserializerClass the deserializerClass to set
+	 */
+	public void setDeserializerClass(Class<? extends Deserializer> deserializerClass) {
+		this.deserializerClass = deserializerClass;
+	}
+
+	/**
+	 * @return the inputFormatClass
+	 */
+	public Class<? extends InputFormat> getInputFormatClass() {
+		return inputFormatClass;
+	}
+
+	/**
+	 * @param inputFormatClass the inputFormatClass to set
+	 */
+	public void setInputFormatClass(Class<? extends InputFormat> inputFormatClass) {
+		this.inputFormatClass = inputFormatClass;
+	}
+
+	/**
+	 * @return the limit
+	 */
+  @explain(displayName="limit")
+	public int getLimit() {
+		return limit;
+	}
+
+	/**
+	 * @param limit the limit to set
+	 */
+	public void setLimit(int limit) {
+		this.limit = limit;
+	}
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/limitDesc.java

@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.plan;
+
+import java.io.Serializable;
+
+@explain(displayName="Limit")
+public class limitDesc implements Serializable {
+  private static final long serialVersionUID = 1L;
+  private int limit;
+  public limitDesc() { }
+  public limitDesc(final int limit) {
+    this.limit = limit;
+  }
+
+  public int getLimit() {
+    return this.limit;
+  }
+  public void setLimit(final int limit) {
+    this.limit=limit;
+  }
+
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/loadFileDesc.java

@@ -19,22 +19,28 @@
 package org.apache.hadoop.hive.ql.plan;
 
 import java.io.Serializable;
+import java.util.List;
+
 import org.apache.hadoop.hive.ql.plan.loadDesc;
 
 public class loadFileDesc extends loadDesc implements Serializable {
   private static final long serialVersionUID = 1L;
   private String targetDir;
   private boolean isDfsDir;
+  // list of columns, comma separated
+  private String  columns;
 
   public loadFileDesc() { }
   public loadFileDesc(
     final String sourceDir,
     final String targetDir,
-    final boolean isDfsDir) {
+    final boolean isDfsDir, 
+    final String  columns) {
 
     super(sourceDir);
     this.targetDir = targetDir;
     this.isDfsDir = isDfsDir;
+    this.columns = columns;
   }
   
   @explain(displayName="destination")
@@ -52,4 +58,18 @@ public class loadFileDesc extends loadDesc implements Serializable {
   public void setIsDfsDir(final boolean isDfsDir) {
     this.isDfsDir = isDfsDir;
   }
+  
+	/**
+	 * @return the columns
+	 */
+	public String getColumns() {
+		return columns;
+	}
+	
+	/**
+	 * @param columns the columns to set
+	 */
+	public void setColumns(String columns) {
+		this.columns = columns;
+	}
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java

@@ -49,6 +49,7 @@ public class mapredWork implements Serializable {
   private Integer numReduceTasks;
   
   private boolean needsTagging;
+  private boolean inferNumReducers;
 
   public mapredWork() { }
   public mapredWork(
@@ -197,4 +198,13 @@ public class mapredWork implements Serializable {
   public void setNeedsTagging(boolean needsTagging) {
     this.needsTagging = needsTagging;
   }
+
+  public boolean getInferNumReducers() {
+    return this.inferNumReducers;
+  }
+  
+  public void setInferNumReducers(boolean inferNumReducers) {
+    this.inferNumReducers = inferNumReducers;
+  }
+
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/reduceSinkDesc.java

@@ -38,33 +38,25 @@ public class reduceSinkDesc implements Serializable {
   // If the value is -1, then data will go to a random reducer 
   private int numPartitionFields;
   
-  public reduceSinkDesc() { }
+  private boolean inferNumReducers;
+  private int numReducers;
 
-  public reduceSinkDesc
-    (final java.util.ArrayList<exprNodeDesc> keyCols,
-     final java.util.ArrayList<exprNodeDesc> valueCols,
-     final int numPartitionFields,
-     final tableDesc keySerializeInfo,
-     final tableDesc valueSerializeInfo) {
-    this.keyCols = keyCols;
-    this.valueCols = valueCols;
-    this.tag = -1;
-    this.numPartitionFields = numPartitionFields;
-    this.keySerializeInfo = keySerializeInfo;
-    this.valueSerializeInfo = valueSerializeInfo;
-  }
+  public reduceSinkDesc() { }
 
   public reduceSinkDesc
     (java.util.ArrayList<exprNodeDesc> keyCols,
      java.util.ArrayList<exprNodeDesc> valueCols,
      int tag,
      int numPartitionFields,
+     int numReducers,
+     boolean inferNumReducers,
      final tableDesc keySerializeInfo,
      final tableDesc valueSerializeInfo) {
     this.keyCols = keyCols;
     this.valueCols = valueCols;
-    assert tag != -1;
     this.tag = tag;
+    this.numReducers = numReducers;
+    this.inferNumReducers = inferNumReducers;
     this.numPartitionFields = numPartitionFields;
     this.keySerializeInfo = keySerializeInfo;
     this.valueSerializeInfo = valueSerializeInfo;
@@ -104,6 +96,20 @@ public class reduceSinkDesc implements Serializable {
     this.tag = tag;
   }
 
+  public boolean getInferNumReducers() {
+    return this.inferNumReducers;
+  }
+  public void setInferNumReducers(boolean inferNumReducers) {
+    this.inferNumReducers = inferNumReducers;
+  }
+
+  public int getNumReducers() {
+    return this.numReducers;
+  }
+  public void setNumReducers(int numReducers) {
+    this.numReducers = numReducers;
+  }
+
   public tableDesc getKeySerializeInfo() {
     return keySerializeInfo;
   }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/showPartitionsDesc.java

@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.plan;
+
+import java.io.Serializable;
+import org.apache.hadoop.fs.Path;
+
+@explain(displayName="Show Partitions")
+public class showPartitionsDesc extends ddlDesc implements Serializable 
+{
+  private static final long serialVersionUID = 1L;
+  String     tabName;
+  Path       resFile;
+  
+  /**
+   * @param tabName Name of the table whose partitions need to be listed
+   * @param resFile File to store the results in
+   */
+  public showPartitionsDesc(String tabName, Path resFile) {
+    this.tabName = tabName;
+    this.resFile = resFile;
+  }
+
+  /**
+   * @return the name of the table
+   */
+  @explain(displayName="table")
+  public String getTabName() {
+    return tabName;
+  }
+
+  /**
+   * @param tabName the table whose partitions have to be listed
+   */
+  public void setTabName(String tabName) {
+    this.tabName = tabName;
+  }
+
+  /**
+   * @return the results file
+   */
+  public Path getResFile() {
+    return resFile;
+  }
+
+  @explain(displayName="result file", normalExplain=false)
+  public String getResFileString() {
+    return getResFile().getName();
+  }
+  /**
+   * @param resFile the results file to be used to return the results
+   */
+  public void setResFile(Path resFile) {
+    this.resFile = resFile;
+  }
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/tableDesc.java

@@ -47,7 +47,7 @@ public class tableDesc implements Serializable {
   public Class<? extends Deserializer> getDeserializerClass() {
     return this.deserializerClass;
   }
-  public void setDeserializerClass(final Class<? extends Deserializer> serdeClass) {
+  public void setDeserializerClass(final Class<? extends SerDe> serdeClass) {
     this.deserializerClass = serdeClass;
   }
   public Class<? extends InputFormat> getInputFileFormatClass() {

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/StructTypeInfo.java

@@ -95,12 +95,14 @@ public class StructTypeInfo extends TypeInfo implements Serializable{
   }
   
   public TypeInfo getStructFieldTypeInfo(String field) {
+    String fieldLowerCase = field.toLowerCase();
     for(int i=0; i<allStructFieldNames.size(); i++) {
-      if (field.equals(allStructFieldNames.get(i))) {
+      if (fieldLowerCase.equals(allStructFieldNames.get(i))) {
         return allStructFieldTypeInfos.get(i);
       }
     }
-    throw new RuntimeException("cannot find field " + field + " in " + allStructFieldNames);
+    throw new RuntimeException("cannot find field " + field + "(lowercase form: " 
+        + fieldLowerCase + ") in " + allStructFieldNames);
     // return null;
   }
   

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java

@@ -38,25 +38,31 @@ public class UDAFAvg extends UDAF {
   }
 
   public boolean aggregate(String o) {
-    mSum += Double.parseDouble(o);
-    mCount ++;
+    if (o != null && !o.isEmpty()) {
+      mSum += Double.parseDouble(o);
+      mCount ++;
+    }
     return true;
   }
   
   public String evaluatePartial() {
-    return new Double(mSum).toString() + '/' + Long.valueOf(mCount).toString();
+    // This is SQL standard - average of zero items should be null.
+    return mCount == 0 ? null : String.valueOf(mSum) + '/' + String.valueOf(mCount);
   }
 
   public boolean aggregatePartial(String o) {
-    int pos = o.indexOf('/');
-    assert(pos != -1);
-    mSum += Double.parseDouble(o.substring(0, pos));
-    mCount += Long.parseLong(o.substring(pos+1));
+    if (o != null && !o.isEmpty()) {
+      int pos = o.indexOf('/');
+      assert(pos != -1);
+      mSum += Double.parseDouble(o.substring(0, pos));
+      mCount += Long.parseLong(o.substring(pos+1));
+    }
     return true;
   }
 
   public String evaluate() {
-    return new Double(mSum / mCount).toString();
+    // This is SQL standard - average of zero items should be null.
+    return mCount == 0 ? null : String.valueOf(mSum / mCount);
   }
 
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFCount.java

@@ -35,7 +35,10 @@ public class UDAFCount extends UDAF {
   }
   
   public boolean aggregate(Object o) {
-    mCount ++;
+    // Our SerDe between map/reduce boundary may convert MetadataTypedSerDe to 
+    if (o != null && !o.equals("")) {
+      mCount ++;
+    }
     return true;
   }
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFMax.java

@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.UDAF;
 public class UDAFMax extends UDAF {
 
   private double mMax;
+  private boolean mEmpty;
   
   public UDAFMax() {
     super();
@@ -33,24 +34,31 @@ public class UDAFMax extends UDAF {
 
   public void init() {
     mMax = 0;
+    mEmpty = true;
   }
 
   public boolean aggregate(String o) {
-    mMax = Math.max(mMax, Double.parseDouble(o));
+    if (o != null && !o.isEmpty()) {
+      if (mEmpty) {
+        mMax = Double.parseDouble(o);
+        mEmpty = false;
+      } else {
+        mMax = Math.max(mMax, Double.parseDouble(o));
+      }
+    }
     return true;
   }
   
   public String evaluatePartial() {
-    return new Double(mMax).toString();
+    return mEmpty ? null : String.valueOf(mMax);
   }
 
   public boolean aggregatePartial(String o) {
-    mMax = Math.max(mMax, Double.parseDouble(o));
-    return true;
+    return aggregate(o);
   }
 
   public String evaluate() {
-    return new Double(mMax).toString();
+    return mEmpty ? null : String.valueOf(mMax);
   }
 
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFMin.java

@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.UDAF;
 public class UDAFMin extends UDAF {
 
   private double mMin;
+  private boolean mEmpty;
   
   public UDAFMin() {
     super();
@@ -33,24 +34,31 @@ public class UDAFMin extends UDAF {
 
   public void init() {
     mMin = 0;
+    mEmpty = true;
   }
 
   public boolean aggregate(String o) {
-    mMin = Math.min(mMin, Double.parseDouble(o));
+    if (o != null && !o.isEmpty()) {
+      if (mEmpty) {
+        mMin = Double.parseDouble(o);
+        mEmpty = false;
+      } else {
+        mMin = Math.min(mMin, Double.parseDouble(o));
+      }
+    }
     return true;
   }
   
   public String evaluatePartial() {
-    return new Double(mMin).toString();
+    return mEmpty ? null : String.valueOf(mMin);
   }
 
   public boolean aggregatePartial(String o) {
-    mMin = Math.min(mMin, Double.parseDouble(o));
-    return true;
+    return aggregate(o);
   }
 
   public String evaluate() {
-    return new Double(mMin).toString();
+    return mEmpty ? null : String.valueOf(mMin);
   }
 
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFSum.java

@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.UDAF;
 public class UDAFSum extends UDAF {
 
   private double mSum;
+  private boolean mEmpty;
   
   public UDAFSum() {
     super();
@@ -33,24 +34,33 @@ public class UDAFSum extends UDAF {
 
   public void init() {
     mSum = 0;
+    mEmpty = true;
   }
 
   public boolean aggregate(String o) {
-    mSum += Double.parseDouble(o);
+    if (o != null && !o.isEmpty()) {
+      mSum += Double.parseDouble(o);
+      mEmpty = false;
+    }
     return true;
   }
   
   public String evaluatePartial() {
-    return new Double(mSum).toString();
+    // This is SQL standard - sum of zero items should be null.
+    return mEmpty ? null : new Double(mSum).toString();
   }
 
   public boolean aggregatePartial(String o) {
-    mSum += Double.parseDouble(o);
+    if (o != null && !o.isEmpty()) {
+      mSum += Double.parseDouble(o);
+      mEmpty = false;
+    }
     return true;
   }
 
   public String evaluate() {
-    return new Double(mSum).toString();
+    // This is SQL standard - sum of zero items should be null.
+    return mEmpty ? null : new Double(mSum).toString();
   }
 
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseCompare.java

@@ -25,7 +25,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 
-public abstract class UDFBaseCompare extends UDF {
+public abstract class UDFBaseCompare implements UDF {
 
   private static Log LOG = LogFactory.getLog(UDFBaseCompare.class.getName());
 
@@ -38,28 +38,26 @@ public abstract class UDFBaseCompare extends UDF {
    *  String to double and the Number to double, and then compare.
    */
   public Boolean evaluate(String a, Number b)  {
-    Boolean r = null;
-    if ((a == null) || (b == null)) {
-      r = null;
-    } else {
-      r = evaluate(Double.valueOf(a), new Double(b.doubleValue()));
+    Double aDouble = null;
+    try {
+      aDouble = Double.valueOf(a);
+    } catch (Exception e){
+      // do nothing: aDouble will be null.
     }
-    // LOG.info("evaluate(" + a + "," + b + ")=" + r);
-    return r;
+    return evaluate(aDouble, new Double(b.doubleValue()));
   }
 
   /** If one of the argument is a String and the other is a Number, convert
    *  String to double and the Number to double, and then compare.
    */
   public Boolean evaluate(Number a, String b)  {
-    Boolean r = null;
-    if ((a == null) || (b == null)) {
-      r = null;
-    } else {
-      r = evaluate(new Double(a.doubleValue()), Double.valueOf(b));
+    Double bDouble = null;
+    try {
+      bDouble = Double.valueOf(b);
+    } catch (Exception e){
+      // do nothing: bDouble will be null.
     }
-    // LOG.info("evaluate(" + a + "," + b + ")=" + r);
-    return r;
+    return evaluate(new Double(a.doubleValue()), bDouble);
   }
   
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorInfoList.java → src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java

@@ -16,28 +16,25 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hive.ql.parse;
+package org.apache.hadoop.hive.ql.udf;
 
-import java.util.*;
-import org.apache.hadoop.hive.ql.parse.OperatorInfo;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.UDF;
 
-/**
- * Implementation of the OperatorInfoList
- *
- **/
+public class UDFCeil implements UDF {
 
-public class OperatorInfoList extends Vector<OperatorInfo> {
-  /**
-   *
-   */
-private static final long serialVersionUID = 1L;
+  private static Log LOG = LogFactory.getLog(UDFCeil.class.getName());
 
-public Object clone() {
-    OperatorInfoList newL = new OperatorInfoList();
-  
-    for(int i=0; i<size(); ++i) {
-      newL.add(i, (OperatorInfo)get(i).clone());
+  public UDFCeil() {
+  }
+
+  public Long evaluate(Double i)  {
+    if (i == null) {
+      return null;
+    } else {
+      return Long.valueOf((long)Math.ceil(i));
     }
-    return newL;
   }
+  
 }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFConcat.java

@@ -21,7 +21,7 @@ package org.apache.hadoop.hive.ql.udf;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 
-public class UDFConcat extends UDF {
+public class UDFConcat implements UDF {
 
   public UDFConcat() {
   }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDefaultSampleHashFn.java

@@ -22,7 +22,7 @@ import org.apache.hadoop.hive.ql.exec.UDF;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
-public class UDFDefaultSampleHashFn extends UDF {
+public class UDFDefaultSampleHashFn implements UDF {
   protected final Log LOG;
 
   public UDFDefaultSampleHashFn() {

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+
+public class UDFFloor implements UDF {
+
+  private static Log LOG = LogFactory.getLog(UDFFloor.class.getName());
+
+  public UDFFloor() {
+  }
+
+  public Long evaluate(Double i)  {
+    if (i == null) {
+      return null;
+    } else {
+      return Long.valueOf((long)Math.floor(i));
+    }
+  }
+
+}

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLTrim.java

@@ -24,7 +24,7 @@ import org.apache.commons.lang.StringUtils;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;
 
-public class UDFLTrim extends UDF {
+public class UDFLTrim implements UDF {
 
   public UDFLTrim() {
   }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java

@@ -24,7 +24,7 @@ import org.apache.hadoop.hive.ql.exec.UDF;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;
 
-public class UDFLike extends UDF {
+public class UDFLike implements UDF {
 
   private static Log LOG = LogFactory.getLog(UDFLike.class.getName());
   private String lastLikePattern = null;

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLower.java

@@ -22,7 +22,7 @@ import org.apache.hadoop.hive.ql.exec.UDF;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;
 
-public class UDFLower extends UDF {
+public class UDFLower implements UDF {
 
   public UDFLower() {
   }

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPAnd.java

@@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 
-public class UDFOPAnd extends UDF {
+public class UDFOPAnd implements UDF {
 
   private static Log LOG = LogFactory.getLog("org.apache.hadoop.hive.ql.udf.UDFOPAnd");
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPBitAnd.java

@@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 
-public class UDFOPBitAnd extends UDF {
+public class UDFOPBitAnd implements UDF {
 
   private static Log LOG = LogFactory.getLog(UDFOPBitAnd.class.getName());
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPBitNot.java

@@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 
-public class UDFOPBitNot extends UDF {
+public class UDFOPBitNot implements UDF {
 
   private static Log LOG = LogFactory.getLog(UDFOPBitNot.class.getName());
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPBitOr.java

@@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 
-public class UDFOPBitOr extends UDF {
+public class UDFOPBitOr implements UDF {
 
   private static Log LOG = LogFactory.getLog("org.apache.hadoop.hive.ql.udf.UDFOPBitOr");
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPBitXor.java

@@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 
-public class UDFOPBitXor extends UDF {
+public class UDFOPBitXor implements UDF {
 
   private static Log LOG = LogFactory.getLog(UDFOPBitXor.class.getName());
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPDivide.java

@@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 
-public class UDFOPDivide extends UDF {
+public class UDFOPDivide implements UDF {
 
   private static Log LOG = LogFactory.getLog("org.apache.hadoop.hive.ql.udf.UDFOPDivide");
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMinus.java

@@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 
-public class UDFOPMinus extends UDF {
+public class UDFOPMinus implements UDF {
 
   private static Log LOG = LogFactory.getLog("org.apache.hadoop.hive.ql.udf.UDFOPMinus");
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java

@@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 
-public class UDFOPMod extends UDF {
+public class UDFOPMod implements UDF {
 
   private static Log LOG = LogFactory.getLog("org.apache.hadoop.hive.ql.udf.UDFOPMod");
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMultiply.java

@@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 
-public class UDFOPMultiply extends UDF {
+public class UDFOPMultiply implements UDF {
 
   private static Log LOG = LogFactory.getLog("org.apache.hadoop.hive.ql.udf.UDFOPMultiply");
 

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java

@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+
+public class UDFOPNegative implements UDF {
+
+  private static Log LOG = LogFactory.getLog(UDFOPNegative.class.getName());
+
+  public UDFOPNegative() {
+  }
+
+  public Byte evaluate(Byte a)  {
+    if (a == null)
+      return null;
+
+    Byte r = Byte.valueOf((byte)-a.byteValue());
+    return r;
+  }
+
+  public Integer evaluate(Integer a)  {
+    if (a == null)
+      return null;
+
+    Integer r = Integer.valueOf(-a.intValue());
+    return r;
+  }
+
+  public Long evaluate(Long a)  {
+    if (a == null)
+      return null;
+
+    Long r = Long.valueOf(-a.longValue());
+    return r;
+  }
+
+  public Float evaluate(Float a)  {
+    if (a == null)
+      return null;
+
+    Float r = Float.valueOf(-a.floatValue());
+    return r;
+  }
+
+  public Double evaluate(Double a)  {
+    if (a == null)
+      return null;
+
+    Double r = Double.valueOf(-a.doubleValue());
+    return r;
+  }
+
+}