HADOOP-4139. Optimize Hive multi group-by.

(Namin Jain via dhruba)



git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@696112 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -574,6 +574,9 @@ Trunk (unreleased changes)
     HADOOP-4129. Changed memory limits of TaskTracker and Tasks to be in
     KiloBytes rather than bytes. (Vinod Kumar Vavilapalli via acmurthy)
 
+    HADOOP-4139. Optimize Hive multi group-by.
+    (Namin Jain via dhruba)
+
 Release 0.18.1 - 2008-09-17
 
   IMPROVEMENTS

src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java

@@ -1026,6 +1026,78 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
     );
   }
 
+  @SuppressWarnings("nls")
+  private OperatorInfo genGroupByPlanGroupByOpForward(
+        QBParseInfo parseInfo, String dest, OperatorInfo forwardOpInfo,
+        groupByDesc.Mode mode)
+    throws SemanticException {
+    RowResolver inputRS  = forwardOpInfo.getRowResolver();
+    RowResolver outputRS = new RowResolver();
+    outputRS.setIsExprResolver(true);
+    ArrayList<exprNodeDesc> groupByKeys = new ArrayList<exprNodeDesc>();
+    ArrayList<aggregationDesc> aggregations = new ArrayList<aggregationDesc>();
+    List<CommonTree> grpByExprs = getGroupByForClause(parseInfo, dest);
+    for (int i = 0; i < grpByExprs.size(); i++) {
+      CommonTree grpbyExpr = grpByExprs.get(i);
+      String text = grpbyExpr.toStringTree();
+      ColumnInfo exprInfo = inputRS.get("",text);
+
+      if (exprInfo == null) {
+        throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(grpbyExpr));
+      }
+
+      groupByKeys.add(new exprNodeColumnDesc(exprInfo.getType(), exprInfo.getInternalName()));
+      String field = (Integer.valueOf(i)).toString();
+      outputRS.put("", text,
+                   new ColumnInfo(field, exprInfo.getType(), exprInfo.getIsVirtual()));
+    }
+
+    // For each aggregation
+    HashMap<String, CommonTree> aggregationTrees = parseInfo
+        .getAggregationExprsForClause(dest);
+    assert (aggregationTrees != null);
+    for (Map.Entry<String, CommonTree> entry : aggregationTrees.entrySet()) {
+      CommonTree value = entry.getValue();
+      String aggName = value.getChild(0).getText();
+      Class<? extends UDAF> aggClass = UDAFRegistry.getUDAF(aggName);
+      assert (aggClass != null);
+      ArrayList<exprNodeDesc> aggParameters = new ArrayList<exprNodeDesc>();
+      ArrayList<Class<?>> aggClasses = new ArrayList<Class<?>>();
+      // 0 is the function name
+      for (int i = 1; i < value.getChildCount(); i++) {
+        String text = value.getChild(i).toStringTree();
+        CommonTree paraExpr = (CommonTree)value.getChild(i);
+        ColumnInfo paraExprInfo = inputRS.get("", text);
+        if (paraExprInfo == null) {
+          throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(paraExpr));
+        }
+
+        String paraExpression = paraExprInfo.getInternalName();
+        assert(paraExpression != null);
+        aggParameters.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExprInfo.getInternalName()));
+        aggClasses.add(paraExprInfo.getType().getPrimitiveClass());
+      }
+
+      if (null == UDAFRegistry.getUDAFMethod(aggName, aggClasses)) {
+        String reason = "Looking for UDAF \"" + aggName + "\" with parameters " + aggClasses;
+        throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((CommonTree)value.getChild(0), reason));
+      }
+      
+      aggregations.add(new aggregationDesc(aggClass, aggParameters,
+          value.getToken().getType() == HiveParser.TOK_FUNCTIONDI));
+      outputRS.put("",value.toStringTree(),
+                                   new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(),
+                                                  String.class, false));  // Everything is a string right now
+    }
+
+    return new OperatorInfo(
+        OperatorFactory.getAndMakeChild(new groupByDesc(mode, groupByKeys, aggregations),
+                                        new RowSchema(outputRS.getColumnInfos()),
+                                        forwardOpInfo.getOp()),
+        outputRS
+    );
+  }
+
   @SuppressWarnings("nls")
   private OperatorInfo genGroupByPlanReduceSinkOperator(QBParseInfo parseInfo,
       String dest, OperatorInfo inputOperatorInfo, int numPartitionFields)
@@ -1094,6 +1166,106 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
     );
   }
 
+  @SuppressWarnings("nls")
+  private OperatorInfo genGroupByPlanReduceSinkOperator(QBParseInfo parseInfo,
+     OperatorInfo input, CommonTree distinctText, TreeSet<String> ks)
+     throws SemanticException {
+    RowResolver inputRS = input.getRowResolver();
+    RowResolver outputRS = new RowResolver();
+    outputRS.setIsExprResolver(true);
+    ArrayList<exprNodeDesc> reduceKeys = new ArrayList<exprNodeDesc>();
+
+    // Spray on distinctText first
+    if (distinctText != null)
+    {
+    	reduceKeys.add(genExprNodeDesc(distinctText, parseInfo.getAlias(), inputRS));
+      String text = distinctText.toStringTree();
+      assert (outputRS.get("", text) == null);
+      outputRS.put("", text,
+        new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(),
+                       String.class, false));
+    }
+    else
+      reduceKeys.add(new exprNodeNullDesc());
+
+    // copy the input row resolver
+    ArrayList<exprNodeDesc> reduceValues = new ArrayList<exprNodeDesc>();
+    Iterator<String> keysIter = inputRS.getTableNames().iterator();
+    while (keysIter.hasNext())
+    {
+      String key = keysIter.next();
+      HashMap<String, ColumnInfo> map = inputRS.getFieldMap(key);
+      Iterator<String> fNamesIter = map.keySet().iterator();
+      while (fNamesIter.hasNext())
+      {
+        String field = fNamesIter.next();
+        ColumnInfo valueInfo = inputRS.get(key, field);
+        
+        if (outputRS.get(key, field) == null) 
+        {
+        	reduceValues.add(new exprNodeColumnDesc(valueInfo.getType(), valueInfo.getInternalName()));
+          outputRS.put(key, field, new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), valueInfo.getType(),
+                                                  valueInfo.getIsVirtual()));
+        }
+      }
+    }
+    
+    for (String dest : ks) {
+      List<CommonTree> grpByExprs = getGroupByForClause(parseInfo, dest);
+
+      // send all the group by expressions
+      for (int i = 0; i < grpByExprs.size(); ++i) {
+        CommonTree grpbyExpr = grpByExprs.get(i);
+        String text = grpbyExpr.toStringTree();
+        if (outputRS.get("", text) == null) {
+          exprNodeDesc grpbyExprNode = genExprNodeDesc(grpbyExpr, parseInfo.getAlias(), inputRS);
+          reduceValues.add(grpbyExprNode);
+          outputRS.put("", text,
+                       new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(),
+                                      grpbyExprNode.getTypeInfo(), false)); 
+        }
+      }
+
+      // send all the aggregation expressions
+      HashMap<String, CommonTree> aggregationTrees = parseInfo.getAggregationExprsForClause(dest);
+      for (Map.Entry<String, CommonTree> entry : aggregationTrees.entrySet()) {
+        CommonTree value = entry.getValue();
+        // 0 is function name
+        for (int i = 1; i < value.getChildCount(); i++) {
+          CommonTree parameter = (CommonTree) value.getChild(i);
+          String text = parameter.toStringTree();
+          if (outputRS.get("",text) == null) {
+            exprNodeDesc pNode = genExprNodeDesc(parameter, parseInfo.getAlias(), inputRS);
+            reduceValues.add(pNode);
+            outputRS.put("", text,
+                         new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(),
+                                        pNode.getTypeInfo(), false));
+          }
+        }
+      }
+    }
+
+    return new OperatorInfo(
+        OperatorFactory.getAndMakeChild(new reduceSinkDesc(reduceKeys, reduceValues, distinctText == null ? -1 : 1),
+                                        new RowSchema(outputRS.getColumnInfos()), input.getOp()),
+        outputRS);
+  }
+
+  @SuppressWarnings("nls")
+  private OperatorInfo genGroupByPlanForwardOperator(QBParseInfo parseInfo, OperatorInfo input)
+      throws SemanticException {
+    RowResolver outputRS = input.getRowResolver();;
+
+    Operator<? extends Serializable> forward = OperatorFactory.get(forwardDesc.class,
+        new RowSchema(outputRS.getColumnInfos()));
+    // set forward operator as child of each of input
+    List<Operator<? extends Serializable>> child = new ArrayList<Operator<? extends Serializable>>();
+    child.add(forward);
+    input.getOp().setChildOperators(child);
+
+    return new OperatorInfo(forward, outputRS);
+  }
+
   @SuppressWarnings("nls")
   private OperatorInfo genGroupByPlanReduceSinkOperator2MR(
       QBParseInfo parseInfo, String dest, OperatorInfo groupByOperatorInfo,
@@ -1298,6 +1470,46 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
     return output;
   }
 
+  /**
+   * Generate a Group-By plan using a 2 map-reduce jobs. The first map-reduce
+   * job has already been constructed. Evaluate partial aggregates first,
+   * followed by actual aggregates. The first map-reduce stage will be 
+   * shared by all groupbys.
+   */
+  @SuppressWarnings("nls")
+  private OperatorInfoList genGroupByPlan3MR(String dest, QB qb, 
+    OperatorInfoList inputList) throws SemanticException {
+
+    // We can assert here that the input list is of size one
+    if (inputList.size() != 1) {
+      throw new SemanticException("Select has more than one inputs");
+    }
+    OperatorInfo inputOperatorInfo = inputList.get(0);
+    QBParseInfo parseInfo = qb.getParseInfo();
+
+    // ////// Generate GroupbyOperator
+    OperatorInfo groupByOperatorInfo = genGroupByPlanGroupByOpForward(parseInfo,
+      dest, inputOperatorInfo, groupByDesc.Mode.PARTIAL1);
+
+    // //////  Generate ReduceSinkOperator2
+    OperatorInfo reduceSinkOperatorInfo2 = genGroupByPlanReduceSinkOperator2MR(
+      parseInfo, dest, groupByOperatorInfo,
+      getGroupByForClause(parseInfo, dest).size());
+
+    // ////// Generate GroupbyOperator2
+    OperatorInfo groupByOperatorInfo2 = genGroupByPlanGroupByOperator2MR(
+      parseInfo, dest, reduceSinkOperatorInfo2);
+
+    // ////// Generate SelectOperator
+    OperatorInfo selectOperatorInfo = genGroupByPlanSelectOperator(parseInfo,
+      dest, groupByOperatorInfo2);
+
+    // ////// Create output
+    OperatorInfoList output = new OperatorInfoList();
+    output.add(selectOperatorInfo);
+    return output;
+  }
+
   @SuppressWarnings("nls")
   private OperatorInfoList genConversionOps(String dest, QB qb,
       OperatorInfoList input) throws SemanticException {
@@ -1870,26 +2082,66 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
     TreeSet<String> ks = new TreeSet<String>();
     ks.addAll(qbp.getClauseNames());
 
-    // Go over all the destination tables
+    String distinctText = null;
+    CommonTree distn = null;
+    OperatorInfoList opList = null;
+    boolean grpBy = false;
+
+    // In case of a multiple group bys, all of them should have the same distinct key
     for (String dest : ks) {
+    	// is it a group by
+      if ((qbp.getAggregationExprsForClause(dest).size() != 0)
+          || (getGroupByForClause(qbp, dest).size() > 0)) {
+        grpBy = true;
+
+        // If there is a distinctFuncExp, add all parameters to the reduceKeys.
+        if (qbp.getDistinctFuncExprForClause(dest) != null) {
+          CommonTree value = qbp.getDistinctFuncExprForClause(dest);
+          if (value.getChildCount() != 2)
+            throw new SemanticException(ErrorMsg.UNSUPPORTED_MULTIPLE_DISTINCTS.getMsg(value));
+          distn = (CommonTree)value.getChild(1);
+          String dist = distn.toStringTree();;
+          if (distinctText == null)
+            distinctText = dist;
+          if (!distinctText.equals(dist))
+            throw new SemanticException(ErrorMsg.UNSUPPORTED_MULTIPLE_DISTINCTS.getMsg(value));
+        }
+      }
+    }
+
+    // In the first stage, copy the input and all the group by expressions
+    // and aggregate paramaters. This can be optimized in the future to only
+    // evaluate expressions that occur frequently
+    if (grpBy) {
+      OperatorInfo reduceSinkOperatorInfo = 
+        genGroupByPlanReduceSinkOperator(qbp, input.get(0), distn, ks);
       
-      OperatorInfoList curr = input;
+      // ////// 2. Generate GroupbyOperator
+      OperatorInfo forwardOperatorInfo = genGroupByPlanForwardOperator(qbp, reduceSinkOperatorInfo);
+      opList = new OperatorInfoList();
+      opList.add(forwardOperatorInfo);
+    }
+
+    // Go over all the destination tables
+    for (String dest : ks) {
+      boolean groupByExpr = false;
+      if (qbp.getAggregationExprsForClause(dest).size() != 0
+          || getGroupByForClause(qbp, dest).size() > 0) 
+        groupByExpr = true;
+
+      OperatorInfoList curr = input;      
+      if (groupByExpr) 
+        curr = opList;
 
       if (qbp.getWhrForClause(dest) != null) {
         curr = genFilterPlan(dest, qb, curr);
       }
 
       if (qbp.getAggregationExprsForClause(dest).size() != 0
-          || getGroupByForClause(qbp, dest).size() > 0) {
-        // We always do 2MR group-by for now.
-        // In the future, we will do 1MR group-by for
-        // 1. Queries without DISTINCT (we will add combiner to 1MR group-by);
-        // 2. Queries with UDAFs that does not support partial aggregations.
-        curr = genGroupByPlan2MR(dest, qb, curr);
-        // curr = genGroupByPlan1MR(dest, qb, curr);
-      } else {
+          || getGroupByForClause(qbp, dest).size() > 0)
+        curr = genGroupByPlan3MR(dest, qb, curr);
+      else 
         curr = genSelectPlan(dest, qb, curr);
-      }
 
       if (qbp.getClusterByForClause(dest) != null) {
         curr = genReduceSinkPlan(dest, qb, curr);

src/contrib/hive/ql/src/test/queries/clientpositive/groupby7.q

@@ -0,0 +1,9 @@
+CREATE TABLE DEST1(key INT, value STRING);
+CREATE TABLE DEST2(key INT, value STRING);
+
+FROM SRC
+INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,4)) GROUP BY SRC.key
+INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, sum(SUBSTR(SRC.value,4)) GROUP BY SRC.key;
+
+SELECT DEST1.* FROM DEST1;
+SELECT DEST1.* FROM DEST2;

src/contrib/hive/ql/src/test/queries/clientpositive/groupby8.q

@@ -0,0 +1,10 @@
+CREATE TABLE DEST1(key INT, value STRING);
+CREATE TABLE DEST2(key INT, value STRING);
+
+FROM SRC
+INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,4)) GROUP BY SRC.key
+INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,4)) GROUP BY SRC.key;
+
+SELECT DEST1.* FROM DEST1;
+SELECT DEST1.* FROM DEST2;
+

src/contrib/hive/ql/src/test/queries/negative/wrong_distinct3.q

@@ -0,0 +1,3 @@
+FROM SRC
+INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,4)) GROUP BY SRC.key
+INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key

src/contrib/hive/ql/src/test/results/clientpositive/groupby7.q.out

@@ -0,0 +1,618 @@
+0	0.0	
+10	10.0	
+100	200.0	
+103	206.0	
+104	208.0	
+105	105.0	
+11	11.0	
+111	111.0	
+113	226.0	
+114	114.0	
+116	116.0	
+118	236.0	
+119	357.0	
+12	24.0	
+120	240.0	
+125	250.0	
+126	126.0	
+128	384.0	
+129	258.0	
+131	131.0	
+133	133.0	
+134	268.0	
+136	136.0	
+137	274.0	
+138	552.0	
+143	143.0	
+145	145.0	
+146	292.0	
+149	298.0	
+15	30.0	
+150	150.0	
+152	304.0	
+153	153.0	
+155	155.0	
+156	156.0	
+157	157.0	
+158	158.0	
+160	160.0	
+162	162.0	
+163	163.0	
+164	328.0	
+165	330.0	
+166	166.0	
+167	501.0	
+168	168.0	
+169	676.0	
+17	17.0	
+170	170.0	
+172	344.0	
+174	348.0	
+175	350.0	
+176	352.0	
+177	177.0	
+178	178.0	
+179	358.0	
+18	36.0	
+180	180.0	
+181	181.0	
+183	183.0	
+186	186.0	
+187	561.0	
+189	189.0	
+19	19.0	
+190	190.0	
+191	382.0	
+192	192.0	
+193	579.0	
+194	194.0	
+195	390.0	
+196	196.0	
+197	394.0	
+199	597.0	
+2	2.0	
+20	20.0	
+200	400.0	
+201	201.0	
+202	202.0	
+203	406.0	
+205	410.0	
+207	414.0	
+208	624.0	
+209	418.0	
+213	426.0	
+214	214.0	
+216	432.0	
+217	434.0	
+218	218.0	
+219	438.0	
+221	442.0	
+222	222.0	
+223	446.0	
+224	448.0	
+226	226.0	
+228	228.0	
+229	458.0	
+230	1150.0	
+233	466.0	
+235	235.0	
+237	474.0	
+238	476.0	
+239	478.0	
+24	48.0	
+241	241.0	
+242	484.0	
+244	244.0	
+247	247.0	
+248	248.0	
+249	249.0	
+252	252.0	
+255	510.0	
+256	512.0	
+257	257.0	
+258	258.0	
+26	52.0	
+260	260.0	
+262	262.0	
+263	263.0	
+265	530.0	
+266	266.0	
+27	27.0	
+272	544.0	
+273	819.0	
+274	274.0	
+275	275.0	
+277	1108.0	
+278	556.0	
+28	28.0	
+280	560.0	
+281	562.0	
+282	564.0	
+283	283.0	
+284	284.0	
+285	285.0	
+286	286.0	
+287	287.0	
+288	576.0	
+289	289.0	
+291	291.0	
+292	292.0	
+296	296.0	
+298	894.0	
+30	30.0	
+302	302.0	
+305	305.0	
+306	306.0	
+307	614.0	
+308	308.0	
+309	618.0	
+310	310.0	
+311	933.0	
+315	315.0	
+316	948.0	
+317	634.0	
+318	954.0	
+321	642.0	
+322	644.0	
+323	323.0	
+325	650.0	
+327	981.0	
+33	33.0	
+331	662.0	
+332	332.0	
+333	666.0	
+335	335.0	
+336	336.0	
+338	338.0	
+339	339.0	
+34	34.0	
+341	341.0	
+342	684.0	
+344	688.0	
+345	345.0	
+348	1740.0	
+35	105.0	
+351	351.0	
+353	706.0	
+356	356.0	
+360	360.0	
+362	362.0	
+364	364.0	
+365	365.0	
+366	366.0	
+367	734.0	
+368	368.0	
+369	1107.0	
+37	74.0	
+373	373.0	
+374	374.0	
+375	375.0	
+377	377.0	
+378	378.0	
+379	379.0	
+382	764.0	
+384	1152.0	
+386	386.0	
+389	389.0	
+392	392.0	
+393	393.0	
+394	394.0	
+395	790.0	
+396	1188.0	
+397	794.0	
+399	798.0	
+4	4.0	
+400	400.0	
+401	2005.0	
+402	402.0	
+403	1209.0	
+404	808.0	
+406	1624.0	
+407	407.0	
+409	1227.0	
+41	41.0	
+411	411.0	
+413	826.0	
+414	828.0	
+417	1251.0	
+418	418.0	
+419	419.0	
+42	84.0	
+421	421.0	
+424	848.0	
+427	427.0	
+429	858.0	
+43	43.0	
+430	1290.0	
+431	1293.0	
+432	432.0	
+435	435.0	
+436	436.0	
+437	437.0	
+438	1314.0	
+439	878.0	
+44	44.0	
+443	443.0	
+444	444.0	
+446	446.0	
+448	448.0	
+449	449.0	
+452	452.0	
+453	453.0	
+454	1362.0	
+455	455.0	
+457	457.0	
+458	916.0	
+459	918.0	
+460	460.0	
+462	924.0	
+463	926.0	
+466	1398.0	
+467	467.0	
+468	1872.0	
+469	2345.0	
+47	47.0	
+470	470.0	
+472	472.0	
+475	475.0	
+477	477.0	
+478	956.0	
+479	479.0	
+480	1440.0	
+481	481.0	
+482	482.0	
+483	483.0	
+484	484.0	
+485	485.0	
+487	487.0	
+489	1956.0	
+490	490.0	
+491	491.0	
+492	984.0	
+493	493.0	
+494	494.0	
+495	495.0	
+496	496.0	
+497	497.0	
+498	1494.0	
+5	15.0	
+51	102.0	
+53	53.0	
+54	54.0	
+57	57.0	
+58	116.0	
+64	64.0	
+65	65.0	
+66	66.0	
+67	134.0	
+69	69.0	
+70	210.0	
+72	144.0	
+74	74.0	
+76	152.0	
+77	77.0	
+78	78.0	
+8	8.0	
+80	80.0	
+82	82.0	
+83	166.0	
+84	168.0	
+85	85.0	
+86	86.0	
+87	87.0	
+9	9.0	
+90	270.0	
+92	92.0	
+95	190.0	
+96	96.0	
+97	194.0	
+98	196.0	
+0	0.0	
+10	10.0	
+100	200.0	
+103	206.0	
+104	208.0	
+105	105.0	
+11	11.0	
+111	111.0	
+113	226.0	
+114	114.0	
+116	116.0	
+118	236.0	
+119	357.0	
+12	24.0	
+120	240.0	
+125	250.0	
+126	126.0	
+128	384.0	
+129	258.0	
+131	131.0	
+133	133.0	
+134	268.0	
+136	136.0	
+137	274.0	
+138	552.0	
+143	143.0	
+145	145.0	
+146	292.0	
+149	298.0	
+15	30.0	
+150	150.0	
+152	304.0	
+153	153.0	
+155	155.0	
+156	156.0	
+157	157.0	
+158	158.0	
+160	160.0	
+162	162.0	
+163	163.0	
+164	328.0	
+165	330.0	
+166	166.0	
+167	501.0	
+168	168.0	
+169	676.0	
+17	17.0	
+170	170.0	
+172	344.0	
+174	348.0	
+175	350.0	
+176	352.0	
+177	177.0	
+178	178.0	
+179	358.0	
+18	36.0	
+180	180.0	
+181	181.0	
+183	183.0	
+186	186.0	
+187	561.0	
+189	189.0	
+19	19.0	
+190	190.0	
+191	382.0	
+192	192.0	
+193	579.0	
+194	194.0	
+195	390.0	
+196	196.0	
+197	394.0	
+199	597.0	
+2	2.0	
+20	20.0	
+200	400.0	
+201	201.0	
+202	202.0	
+203	406.0	
+205	410.0	
+207	414.0	
+208	624.0	
+209	418.0	
+213	426.0	
+214	214.0	
+216	432.0	
+217	434.0	
+218	218.0	
+219	438.0	
+221	442.0	
+222	222.0	
+223	446.0	
+224	448.0	
+226	226.0	
+228	228.0	
+229	458.0	
+230	1150.0	
+233	466.0	
+235	235.0	
+237	474.0	
+238	476.0	
+239	478.0	
+24	48.0	
+241	241.0	
+242	484.0	
+244	244.0	
+247	247.0	
+248	248.0	
+249	249.0	
+252	252.0	
+255	510.0	
+256	512.0	
+257	257.0	
+258	258.0	
+26	52.0	
+260	260.0	
+262	262.0	
+263	263.0	
+265	530.0	
+266	266.0	
+27	27.0	
+272	544.0	
+273	819.0	
+274	274.0	
+275	275.0	
+277	1108.0	
+278	556.0	
+28	28.0	
+280	560.0	
+281	562.0	
+282	564.0	
+283	283.0	
+284	284.0	
+285	285.0	
+286	286.0	
+287	287.0	
+288	576.0	
+289	289.0	
+291	291.0	
+292	292.0	
+296	296.0	
+298	894.0	
+30	30.0	
+302	302.0	
+305	305.0	
+306	306.0	
+307	614.0	
+308	308.0	
+309	618.0	
+310	310.0	
+311	933.0	
+315	315.0	
+316	948.0	
+317	634.0	
+318	954.0	
+321	642.0	
+322	644.0	
+323	323.0	
+325	650.0	
+327	981.0	
+33	33.0	
+331	662.0	
+332	332.0	
+333	666.0	
+335	335.0	
+336	336.0	
+338	338.0	
+339	339.0	
+34	34.0	
+341	341.0	
+342	684.0	
+344	688.0	
+345	345.0	
+348	1740.0	
+35	105.0	
+351	351.0	
+353	706.0	
+356	356.0	
+360	360.0	
+362	362.0	
+364	364.0	
+365	365.0	
+366	366.0	
+367	734.0	
+368	368.0	
+369	1107.0	
+37	74.0	
+373	373.0	
+374	374.0	
+375	375.0	
+377	377.0	
+378	378.0	
+379	379.0	
+382	764.0	
+384	1152.0	
+386	386.0	
+389	389.0	
+392	392.0	
+393	393.0	
+394	394.0	
+395	790.0	
+396	1188.0	
+397	794.0	
+399	798.0	
+4	4.0	
+400	400.0	
+401	2005.0	
+402	402.0	
+403	1209.0	
+404	808.0	
+406	1624.0	
+407	407.0	
+409	1227.0	
+41	41.0	
+411	411.0	
+413	826.0	
+414	828.0	
+417	1251.0	
+418	418.0	
+419	419.0	
+42	84.0	
+421	421.0	
+424	848.0	
+427	427.0	
+429	858.0	
+43	43.0	
+430	1290.0	
+431	1293.0	
+432	432.0	
+435	435.0	
+436	436.0	
+437	437.0	
+438	1314.0	
+439	878.0	
+44	44.0	
+443	443.0	
+444	444.0	
+446	446.0	
+448	448.0	
+449	449.0	
+452	452.0	
+453	453.0	
+454	1362.0	
+455	455.0	
+457	457.0	
+458	916.0	
+459	918.0	
+460	460.0	
+462	924.0	
+463	926.0	
+466	1398.0	
+467	467.0	
+468	1872.0	
+469	2345.0	
+47	47.0	
+470	470.0	
+472	472.0	
+475	475.0	
+477	477.0	
+478	956.0	
+479	479.0	
+480	1440.0	
+481	481.0	
+482	482.0	
+483	483.0	
+484	484.0	
+485	485.0	
+487	487.0	
+489	1956.0	
+490	490.0	
+491	491.0	
+492	984.0	
+493	493.0	
+494	494.0	
+495	495.0	
+496	496.0	
+497	497.0	
+498	1494.0	
+5	15.0	
+51	102.0	
+53	53.0	
+54	54.0	
+57	57.0	
+58	116.0	
+64	64.0	
+65	65.0	
+66	66.0	
+67	134.0	
+69	69.0	
+70	210.0	
+72	144.0	
+74	74.0	
+76	152.0	
+77	77.0	
+78	78.0	
+8	8.0	
+80	80.0	
+82	82.0	
+83	166.0	
+84	168.0	
+85	85.0	
+86	86.0	
+87	87.0	
+9	9.0	
+90	270.0	
+92	92.0	
+95	190.0	
+96	96.0	
+97	194.0	
+98	196.0	

src/contrib/hive/ql/src/test/results/clientpositive/groupby8.q.out

@@ -0,0 +1,618 @@
+0	1	
+10	1	
+100	1	
+103	1	
+104	1	
+105	1	
+11	1	
+111	1	
+113	1	
+114	1	
+116	1	
+118	1	
+119	1	
+12	1	
+120	1	
+125	1	
+126	1	
+128	1	
+129	1	
+131	1	
+133	1	
+134	1	
+136	1	
+137	1	
+138	1	
+143	1	
+145	1	
+146	1	
+149	1	
+15	1	
+150	1	
+152	1	
+153	1	
+155	1	
+156	1	
+157	1	
+158	1	
+160	1	
+162	1	
+163	1	
+164	1	
+165	1	
+166	1	
+167	1	
+168	1	
+169	1	
+17	1	
+170	1	
+172	1	
+174	1	
+175	1	
+176	1	
+177	1	
+178	1	
+179	1	
+18	1	
+180	1	
+181	1	
+183	1	
+186	1	
+187	1	
+189	1	
+19	1	
+190	1	
+191	1	
+192	1	
+193	1	
+194	1	
+195	1	
+196	1	
+197	1	
+199	1	
+2	1	
+20	1	
+200	1	
+201	1	
+202	1	
+203	1	
+205	1	
+207	1	
+208	1	
+209	1	
+213	1	
+214	1	
+216	1	
+217	1	
+218	1	
+219	1	
+221	1	
+222	1	
+223	1	
+224	1	
+226	1	
+228	1	
+229	1	
+230	1	
+233	1	
+235	1	
+237	1	
+238	1	
+239	1	
+24	1	
+241	1	
+242	1	
+244	1	
+247	1	
+248	1	
+249	1	
+252	1	
+255	1	
+256	1	
+257	1	
+258	1	
+26	1	
+260	1	
+262	1	
+263	1	
+265	1	
+266	1	
+27	1	
+272	1	
+273	1	
+274	1	
+275	1	
+277	1	
+278	1	
+28	1	
+280	1	
+281	1	
+282	1	
+283	1	
+284	1	
+285	1	
+286	1	
+287	1	
+288	1	
+289	1	
+291	1	
+292	1	
+296	1	
+298	1	
+30	1	
+302	1	
+305	1	
+306	1	
+307	1	
+308	1	
+309	1	
+310	1	
+311	1	
+315	1	
+316	1	
+317	1	
+318	1	
+321	1	
+322	1	
+323	1	
+325	1	
+327	1	
+33	1	
+331	1	
+332	1	
+333	1	
+335	1	
+336	1	
+338	1	
+339	1	
+34	1	
+341	1	
+342	1	
+344	1	
+345	1	
+348	1	
+35	1	
+351	1	
+353	1	
+356	1	
+360	1	
+362	1	
+364	1	
+365	1	
+366	1	
+367	1	
+368	1	
+369	1	
+37	1	
+373	1	
+374	1	
+375	1	
+377	1	
+378	1	
+379	1	
+382	1	
+384	1	
+386	1	
+389	1	
+392	1	
+393	1	
+394	1	
+395	1	
+396	1	
+397	1	
+399	1	
+4	1	
+400	1	
+401	1	
+402	1	
+403	1	
+404	1	
+406	1	
+407	1	
+409	1	
+41	1	
+411	1	
+413	1	
+414	1	
+417	1	
+418	1	
+419	1	
+42	1	
+421	1	
+424	1	
+427	1	
+429	1	
+43	1	
+430	1	
+431	1	
+432	1	
+435	1	
+436	1	
+437	1	
+438	1	
+439	1	
+44	1	
+443	1	
+444	1	
+446	1	
+448	1	
+449	1	
+452	1	
+453	1	
+454	1	
+455	1	
+457	1	
+458	1	
+459	1	
+460	1	
+462	1	
+463	1	
+466	1	
+467	1	
+468	1	
+469	1	
+47	1	
+470	1	
+472	1	
+475	1	
+477	1	
+478	1	
+479	1	
+480	1	
+481	1	
+482	1	
+483	1	
+484	1	
+485	1	
+487	1	
+489	1	
+490	1	
+491	1	
+492	1	
+493	1	
+494	1	
+495	1	
+496	1	
+497	1	
+498	1	
+5	1	
+51	1	
+53	1	
+54	1	
+57	1	
+58	1	
+64	1	
+65	1	
+66	1	
+67	1	
+69	1	
+70	1	
+72	1	
+74	1	
+76	1	
+77	1	
+78	1	
+8	1	
+80	1	
+82	1	
+83	1	
+84	1	
+85	1	
+86	1	
+87	1	
+9	1	
+90	1	
+92	1	
+95	1	
+96	1	
+97	1	
+98	1	
+0	1	
+10	1	
+100	1	
+103	1	
+104	1	
+105	1	
+11	1	
+111	1	
+113	1	
+114	1	
+116	1	
+118	1	
+119	1	
+12	1	
+120	1	
+125	1	
+126	1	
+128	1	
+129	1	
+131	1	
+133	1	
+134	1	
+136	1	
+137	1	
+138	1	
+143	1	
+145	1	
+146	1	
+149	1	
+15	1	
+150	1	
+152	1	
+153	1	
+155	1	
+156	1	
+157	1	
+158	1	
+160	1	
+162	1	
+163	1	
+164	1	
+165	1	
+166	1	
+167	1	
+168	1	
+169	1	
+17	1	
+170	1	
+172	1	
+174	1	
+175	1	
+176	1	
+177	1	
+178	1	
+179	1	
+18	1	
+180	1	
+181	1	
+183	1	
+186	1	
+187	1	
+189	1	
+19	1	
+190	1	
+191	1	
+192	1	
+193	1	
+194	1	
+195	1	
+196	1	
+197	1	
+199	1	
+2	1	
+20	1	
+200	1	
+201	1	
+202	1	
+203	1	
+205	1	
+207	1	
+208	1	
+209	1	
+213	1	
+214	1	
+216	1	
+217	1	
+218	1	
+219	1	
+221	1	
+222	1	
+223	1	
+224	1	
+226	1	
+228	1	
+229	1	
+230	1	
+233	1	
+235	1	
+237	1	
+238	1	
+239	1	
+24	1	
+241	1	
+242	1	
+244	1	
+247	1	
+248	1	
+249	1	
+252	1	
+255	1	
+256	1	
+257	1	
+258	1	
+26	1	
+260	1	
+262	1	
+263	1	
+265	1	
+266	1	
+27	1	
+272	1	
+273	1	
+274	1	
+275	1	
+277	1	
+278	1	
+28	1	
+280	1	
+281	1	
+282	1	
+283	1	
+284	1	
+285	1	
+286	1	
+287	1	
+288	1	
+289	1	
+291	1	
+292	1	
+296	1	
+298	1	
+30	1	
+302	1	
+305	1	
+306	1	
+307	1	
+308	1	
+309	1	
+310	1	
+311	1	
+315	1	
+316	1	
+317	1	
+318	1	
+321	1	
+322	1	
+323	1	
+325	1	
+327	1	
+33	1	
+331	1	
+332	1	
+333	1	
+335	1	
+336	1	
+338	1	
+339	1	
+34	1	
+341	1	
+342	1	
+344	1	
+345	1	
+348	1	
+35	1	
+351	1	
+353	1	
+356	1	
+360	1	
+362	1	
+364	1	
+365	1	
+366	1	
+367	1	
+368	1	
+369	1	
+37	1	
+373	1	
+374	1	
+375	1	
+377	1	
+378	1	
+379	1	
+382	1	
+384	1	
+386	1	
+389	1	
+392	1	
+393	1	
+394	1	
+395	1	
+396	1	
+397	1	
+399	1	
+4	1	
+400	1	
+401	1	
+402	1	
+403	1	
+404	1	
+406	1	
+407	1	
+409	1	
+41	1	
+411	1	
+413	1	
+414	1	
+417	1	
+418	1	
+419	1	
+42	1	
+421	1	
+424	1	
+427	1	
+429	1	
+43	1	
+430	1	
+431	1	
+432	1	
+435	1	
+436	1	
+437	1	
+438	1	
+439	1	
+44	1	
+443	1	
+444	1	
+446	1	
+448	1	
+449	1	
+452	1	
+453	1	
+454	1	
+455	1	
+457	1	
+458	1	
+459	1	
+460	1	
+462	1	
+463	1	
+466	1	
+467	1	
+468	1	
+469	1	
+47	1	
+470	1	
+472	1	
+475	1	
+477	1	
+478	1	
+479	1	
+480	1	
+481	1	
+482	1	
+483	1	
+484	1	
+485	1	
+487	1	
+489	1	
+490	1	
+491	1	
+492	1	
+493	1	
+494	1	
+495	1	
+496	1	
+497	1	
+498	1	
+5	1	
+51	1	
+53	1	
+54	1	
+57	1	
+58	1	
+64	1	
+65	1	
+66	1	
+67	1	
+69	1	
+70	1	
+72	1	
+74	1	
+76	1	
+77	1	
+78	1	
+8	1	
+80	1	
+82	1	
+83	1	
+84	1	
+85	1	
+86	1	
+87	1	
+9	1	
+90	1	
+92	1	
+95	1	
+96	1	
+97	1	
+98	1	

src/contrib/hive/ql/src/test/results/compiler/errors/wrong_distinct3.q.out

@@ -0,0 +1,2 @@
+Semantic Exception: 
+line 3:45 DISTINCT on Different Columns not Supported 5