AMBARI-10810. Make Hive theme changes to reduce number of configs (srimanth)

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml

@@ -356,21 +356,13 @@ limitations under the License.
       <type>value-list</type>
       <entries>
         <entry>
-          <value>ORCFile</value>
+          <value>ORC</value>
           <description>The Optimized Row Columnar (ORC) file format provides a highly efficient way to store Hive data. It was designed to overcome limitations of the other Hive file formats. Using ORC files improves performance when Hive is reading, writing, and processing data.</description>
         </entry>
         <entry>
           <value>TextFile</value>
           <description>Text file format saves Hive data as normal text.</description>
         </entry>
-        <entry>
-          <value>SequenceFile</value>
-          <description>Compressed text file format saves Hive data in CompressedStorage.</description>
-        </entry>
-        <entry>
-          <value>RCfile</value>
-          <description>Short of Record Columnar File, are flat files consisting of binary key/value pairs.</description>
-        </entry>
       </entries>
     </value-attributes>
   </property>

ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml

@@ -175,7 +175,7 @@ limitations under the License.
     <name>hive.exec.reducers.bytes.per.reducer</name>
     <value>67108864</value>
     <description>size per reducer.The default is 256Mb, i.e if the input size is 1G, it will use 4 reducers.</description>
-    <display-name>Memory</display-name>
+    <display-name>Data per Reducer</display-name>
     <value-attributes>
       <type>int</type>
       <minimum>64</minimum>
@@ -1096,7 +1096,7 @@ limitations under the License.
         </entry>
         <entry>
           <value>LDAP</value>
-          <label>Lightweight Directory Access Protocol</label>
+          <label>LDAP</label>
         </entry>
         <entry>
           <value>KERBEROS</value>
@@ -1104,7 +1104,11 @@ limitations under the License.
         </entry>
         <entry>
           <value>PAM</value>
-          <label>Pluggable Authentication Modules</label>
+          <label>PAM</label>
+        </entry>
+        <entry>
+          <value>CUSTOM</value>
+          <label>Custom</label>
         </entry>
       </entries>
       <selection-cardinality>1</selection-cardinality>
@@ -1399,6 +1403,12 @@ limitations under the License.
       <unit>B</unit>
       <step-increment></step-increment>
     </value-attributes>
+    <depends-on>
+      <property>
+        <type>hive-site</type>
+        <name>hive.tez.container.size</name>
+      </property>
+    </depends-on>
   </property>
 
   <property>

ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json

@@ -13,121 +13,83 @@
               "tab-rows": 6,
               "tab-columns": 3,
               "sections": [
-                {
-                  "name": "optimization",
-                  "display-name": "Optimization",
+                 {
+                  "name": "acid-transactions",
+                  "display-name": "ACID Transactions",
                   "row-index": "0",
                   "column-index": "0",
-                  "row-span": "3",
-                  "column-span": "2",
-                  "section-columns": "2",
-                  "section-rows": "4",
+                  "row-span": "1",
+                  "column-span": "1",
+                  "section-columns": "1",
+                  "section-rows": "1",
                   "subsections": [
                     {
-                      "name": "optimization-row1-col1",
-                      "display-name": "Storage",
-                      "row-index": "0",
-                      "column-index": "0",
-                      "row-span": "2",
-                      "column-span": "1"
-                    },
-                    {
-                      "name": "optimization-row1-col2-1",
-                      "display-name": "",
+                      "name": "acid-transactions-row1-col1-1",
                       "row-index": "0",
-                      "column-index": "1",
-                      "row-span": "1",
-                      "column-span": "1"
-                    },
-                    {
-                      "name": "optimization-row1-col2-2",
-                      "display-name": "",
-                      "row-index": "1",
-                      "column-index": "1",
-                      "row-span": "1",
-                      "column-span": "1",
-                      "border": true
-                    },
-                    {
-                      "name": "optimization-row2-col1",
-                      "display-name": "Optimization Vectorization",
-                      "row-index": "2",
                       "column-index": "0",
                       "row-span": "1",
                       "column-span": "1"
-                    },
-                    {
-                      "name": "optimization-row2-col2",
-                      "display-name": "",
-                      "row-index": "2",
-                      "column-index": "1",
-                      "row-span": "1",
-                      "column-span": "1"
-                    },
+                    }
+                  ]
+                },
+                {
+                  "name": "interactive-query",
+                  "display-name": "Interactive Query",
+                  "row-index": "0",
+                  "column-index": "1",
+                  "row-span": "1",
+                  "column-span": "1",
+                  "section-columns": "1",
+                  "section-rows": "1",
+                  "subsections": [
                     {
-                      "name": "optimization-row3-col1",
-                      "display-name": "Memory",
-                      "row-index": "3",
+                      "name": "interactive-query-row1-col1",
+                      "row-index": "0",
                       "column-index": "0",
                       "row-span": "1",
                       "column-span": "1"
-                    },
-                    {
-                      "name": "optimization-row3-col2",
-                      "display-name": "",
-                      "row-index": "3",
-                      "column-index": "1",
-                      "row-span": "1",
-                      "column-span": "1"
                     }
                   ]
                 },
                 {
-                  "name": "acid-transactions",
-                  "display-name": "Acid Transactions",
+                  "name": "security",
+                  "display-name": "Security",
                   "row-index": "0",
                   "column-index": "2",
-                  "row-span": "2",
+                  "row-span": "1",
                   "column-span": "1",
                   "section-columns": "1",
-                  "section-rows": "2",
+                  "section-rows": "1",
                   "subsections": [
                     {
-                      "name": "acid-transactions-row1-col1-1",
+                      "name": "security-row1-col1",
                       "row-index": "0",
                       "column-index": "0",
                       "row-span": "1",
                       "column-span": "1"
-                    },
-                    {
-                      "name": "acid-transactions-row1-col1-2",
-                      "row-index": "1",
-                      "column-index": "0",
-                      "row-span": "1",
-                      "column-span": "1"
                     }
                   ]
                 },
                 {
-                  "name": "performance",
-                  "display-name": "Performance",
-                  "row-index": "3",
+                  "name": "optimization",
+                  "display-name": "Optimization",
+                  "row-index": "1",
                   "column-index": "0",
-                  "row-span": "3",
-                  "column-span": "2",
-                  "section-columns": "2",
+                  "row-span": "1",
+                  "column-span": "3",
+                  "section-columns": "3",
                   "section-rows": "2",
                   "subsections": [
                     {
-                      "name": "performance-row1-col1",
-                      "display-name": "TEZ Engine",
+                      "name": "optimization-row1-col1",
+                      "display-name": "Tez",
                       "row-index": "0",
                       "column-index": "0",
                       "row-span": "1",
                       "column-span": "1"
                     },
                     {
-                      "name": "performance-row1-col2",
+                      "name": "optimization-row1-col2",
                       "display-name": "",
                       "row-index": "0",
                       "column-index": "1",
@@ -135,55 +97,35 @@
                       "column-span": "1"
                     },
                     {
-                      "name": "performance-row2-col1",
+                      "name": "optimization-row1-col3",
+                      "display-name": "CBO",
+                      "row-index": "0",
+                      "column-index": "2",
+                      "row-span": "1",
+                      "column-span": "1"
+                    },
+                    {
+                      "name": "optimization-row2-col1",
+                      "display-name": "Storage",
                       "row-index": "1",
                       "column-index": "0",
                       "row-span": "1",
                       "column-span": "1"
                     },
                     {
-                      "name": "performance-row2-col2",
+                      "name": "optimization-row2-col2",
+                      "display-name": "",
                       "row-index": "1",
                       "column-index": "1",
                       "row-span": "1",
                       "column-span": "1"
-                    }
-                  ]
-                },
-                {
-                  "name": "interactive-query",
-                  "display-name": "Interactive Query",
-                  "row-index": "2",
-                  "column-index": "2",
-                  "row-span": "2",
-                  "column-span": "1",
-                  "section-columns": "1",
-                  "section-rows": "2",
-                  "subsections": [
-                    {
-                      "name": "interactive-query-row1-col1",
-                      "row-index": "0",
-                      "column-index": "0",
-                      "row-span": "2",
-                      "column-span": "1"
-                    }
-                  ]
-                },
-                {
-                  "name": "security",
-                  "display-name": "Security",
-                  "row-index": "4",
-                  "column-index": "2",
-                  "row-span": "2",
-                  "column-span": "1",
-                  "section-columns": "1",
-                  "section-rows": "2",
-                  "subsections": [
+                    },
                     {
-                      "name": "security-row1-col1",
-                      "row-index": "0",
-                      "column-index": "0",
-                      "row-span": "2",
+                      "name": "optimization-row2-col3",
+                      "display-name": "Memory",
+                      "row-index": "1",
+                      "column-index": "2",
+                      "row-span": "1",
                       "column-span": "1"
                     }
                   ]
@@ -197,66 +139,30 @@
     "placement": {
       "configuration-layout": "default",
       "configs": [
-        {
-          "config": "hive-site/hive.default.fileformat",
-          "subsection-name": "optimization-row1-col1"
-        },
         {
           "config": "hive-site/hive.exec.orc.default.stripe.size",
-          "subsection-name": "optimization-row1-col1"
+          "subsection-name": "optimization-row2-col1"
         },
         {
           "config": "hive-site/hive.exec.orc.default.compress",
-          "subsection-name": "optimization-row1-col2-1"
-        },
-        {
-          "config": "hive-env/hive_exec_orc_storage_strategy",
-          "subsection-name": "optimization-row1-col2-2"
-        },
-        {
-          "config": "hive-site/hive.optimize.index.filter",
-          "subsection-name": "optimization-row1-col1"
-        },
-        {
-          "config": "hive-site/hive.optimize.sort.dynamic.partition",
-          "subsection-name": "optimization-row1-col1"
-        },
-        {
-          "config": "hive-site/hive.exec.orc.encoding.strategy",
-          "subsection-name": "optimization-row1-col2-2"
-        },
-        {
-          "config": "hive-site/hive.exec.orc.compression.strategy",
-          "subsection-name": "optimization-row1-col2-2"
-        },
-        {
-          "config": "hive-site/hive.vectorized.execution.enabled",
           "subsection-name": "optimization-row2-col1"
         },
         {
-          "config": "hive-site/hive.vectorized.execution.reduce.enabled",
+          "config": "hive-env/hive_exec_orc_storage_strategy",
           "subsection-name": "optimization-row2-col2"
         },
         {
           "config": "hive-site/hive.auto.convert.join.noconditionaltask.size",
-          "subsection-name": "optimization-row3-col1"
+          "subsection-name": "optimization-row2-col3"
         },
         {
           "config": "hive-site/hive.exec.reducers.bytes.per.reducer",
-          "subsection-name": "optimization-row3-col2"
+          "subsection-name": "optimization-row2-col3"
         },
         {
           "config": "hive-env/hive_txn_acid",
           "subsection-name": "acid-transactions-row1-col1-1"
         },
-        {
-          "config": "hive-site/hive.txn.manager",
-          "subsection-name": "acid-transactions-row1-col1-1"
-        },
-        {
-          "config": "hive-site/hive.support.concurrency",
-          "subsection-name": "acid-transactions-row1-col1-1"
-        },
         {
           "config": "hive-site/hive.compactor.initiator.on",
           "subsection-name": "acid-transactions-row1-col1-1"
@@ -265,57 +171,29 @@
           "config": "hive-site/hive.compactor.worker.threads",
           "subsection-name": "acid-transactions-row1-col1-1"
         },
-        {
-          "config": "hive-site/hive.enforce.bucketing",
-          "subsection-name": "acid-transactions-row1-col1-2"
-        },
-        {
-          "config": "hive-site/hive.exec.dynamic.partition.mode",
-          "subsection-name": "acid-transactions-row1-col1-2"
-        },
         {
           "config": "hive-site/hive.execution.engine",
-          "subsection-name": "performance-row1-col1"
-        },
-        {
-          "config": "hive-env/hive_timeline_logging_enabled",
-          "subsection-name": "performance-row1-col1"
+          "subsection-name": "optimization-row1-col1"
         },
         {
           "config": "hive-site/hive.tez.container.size",
-          "subsection-name": "performance-row1-col1"
+          "subsection-name": "optimization-row1-col1"
         },
         {
           "config": "hive-site/hive.prewarm.enabled",
-          "subsection-name": "performance-row1-col1"
+          "subsection-name": "optimization-row1-col2"
         },
         {
           "config": "hive-site/hive.prewarm.numcontainers",
-          "subsection-name": "performance-row1-col2"
-        },
-        {
-          "config": "hive-site/hive.tez.auto.reducer.parallelism",
-          "subsection-name": "performance-row1-col2"
-        },
-        {
-          "config": "hive-site/hive.tez.dynamic.partition.pruning",
-          "subsection-name": "performance-row1-col2"
+          "subsection-name": "optimization-row1-col2"
         },
         {
           "config": "hive-site/hive.cbo.enable",
-          "subsection-name": "performance-row2-col1"
-        },
-        {
-          "config": "hive-site/hive.stats.fetch.partition.stats",
-          "subsection-name": "performance-row2-col1"
+          "subsection-name": "optimization-row1-col3"
         },
         {
           "config": "hive-site/hive.stats.fetch.column.stats",
-          "subsection-name": "performance-row2-col1"
-        },
-        {
-          "config": "hive-site/hive.compute.query.using.stats",
-          "subsection-name": "performance-row2-col2"
+          "subsection-name": "optimization-row1-col3"
         },
         {
           "config": "hiveserver2-site/hive.server2.tez.default.queues",
@@ -329,10 +207,6 @@
           "config": "hiveserver2-site/hive.server2.tez.sessions.per.default.queue",
           "subsection-name": "interactive-query-row1-col1"
         },
-        {
-          "config": "hiveserver2-site/hive.server2.enable.doAs",
-          "subsection-name": "interactive-query-row1-col1"
-        },
         {
           "config": "hiveserver2-site/tez.session.am.dag.submit.timeout.secs",
           "subsection-name": "interactive-query-row1-col1"
@@ -342,11 +216,7 @@
           "subsection-name": "security-row1-col1"
         },
         {
-          "config": "hive-site/hive.security.authorization.enabled",
-          "subsection-name": "security-row1-col1"
-        },
-        {
-          "config": "hive-site/hive.security.metastore.authorization.manager",
+          "config": "hiveserver2-site/hive.server2.enable.doAs",
           "subsection-name": "security-row1-col1"
         },
         {
@@ -360,12 +230,6 @@
       ]
     },
     "widgets": [
-      {
-        "config": "hive-site/hive.default.fileformat",
-        "widget": {
-          "type": "combo"
-        }
-      },
       {
         "config": "hive-site/hive.exec.orc.default.stripe.size",
         "widget": {
@@ -383,48 +247,12 @@
           "type": "combo"
         }
       },
-      {
-        "config": "hive-site/hive.optimize.index.filter",
-        "widget": {
-          "type": "toggle"
-        }
-      },
-      {
-        "config": "hive-site/hive.optimize.sort.dynamic.partition",
-        "widget": {
-          "type": "toggle"
-        }
-      },
-      {
-        "config": "hive-site/hive.exec.orc.encoding.strategy",
-        "widget": {
-          "type": "toggle"
-        }
-      },
       {
         "config": "hive-env/hive_exec_orc_storage_strategy",
         "widget": {
           "type": "toggle"
         }
       },
-      {
-        "config": "hive-site/hive.exec.orc.compression.strategy",
-        "widget": {
-          "type": "toggle"
-        }
-      },
-      {
-        "config": "hive-site/hive.vectorized.execution.enabled",
-        "widget": {
-          "type": "toggle"
-        }
-      },
-      {
-        "config": "hive-site/hive.vectorized.execution.reduce.enabled",
-        "widget": {
-          "type": "toggle"
-        }
-      },
       {
         "config": "hive-site/hive.auto.convert.join.noconditionaltask.size",
         "widget": {
@@ -453,24 +281,6 @@
           "type": "toggle"
         }
       },
-      {
-        "config": "hive-site/hive.txn.manager",
-        "widget": {
-          "type": "combo"
-        }
-      },
-      {
-        "config": "hive-site/hive.security.metastore.authorization.manager",
-        "widget": {
-          "type": "string"
-        }
-      },
-      {
-        "config": "hive-site/hive.support.concurrency",
-        "widget": {
-          "type": "toggle"
-        }
-      },
       {
         "config": "hive-site/hive.compactor.initiator.on",
         "widget": {
@@ -488,18 +298,6 @@
           ]
         }
       },
-      {
-        "config": "hive-site/hive.enforce.bucketing",
-        "widget": {
-          "type": "toggle"
-        }
-      },
-      {
-        "config": "hive-site/hive.exec.dynamic.partition.mode",
-        "widget": {
-          "type": "toggle"
-        }
-      },
       {
         "config": "hive-site/hive.execution.engine",
         "widget": {
@@ -534,42 +332,18 @@
           ]
         }
       },
-      {
-        "config": "hive-site/hive.tez.auto.reducer.parallelism",
-        "widget": {
-          "type": "toggle"
-        }
-      },
-      {
-        "config": "hive-site/hive.tez.dynamic.partition.pruning",
-        "widget": {
-          "type": "toggle"
-        }
-      },
       {
         "config": "hive-site/hive.cbo.enable",
         "widget": {
           "type": "toggle"
         }
       },
-      {
-        "config": "hive-site/hive.stats.fetch.partition.stats",
-        "widget": {
-          "type": "toggle"
-        }
-      },
       {
         "config": "hive-site/hive.stats.fetch.column.stats",
         "widget": {
           "type": "toggle"
         }
       },
-      {
-        "config": "hive-site/hive.compute.query.using.stats",
-        "widget": {
-          "type": "toggle"
-        }
-      },
       {
         "config": "hiveserver2-site/hive.server2.tez.initialize.default.sessions",
         "widget": {
@@ -610,12 +384,6 @@
           "type": "combo"
         }
       },
-      {
-        "config": "hive-site/hive.security.authorization.enabled",
-        "widget": {
-          "type": "toggle"
-        }
-      },
       {
         "config": "hive-site/hive.server2.authentication",
         "widget": {
@@ -633,12 +401,6 @@
         "widget": {
           "type": "list"
         }
-      },
-      {
-        "config": "hive-env/hive_timeline_logging_enabled",
-        "widget": {
-          "type": "toggle"
-        }
       }
     ]
   }

ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -233,7 +233,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     putHiveSiteProperty("hive.vectorized.execution.reduce.enabled", "false")
 
     # Memory
-    putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", "2147483648")
     putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864")
 
     # Transactions
@@ -297,6 +296,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       "yarn.scheduler.minimum-allocation-mb" in configurations["yarn-site"]["properties"]:
       container_size = configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]
     putHiveSiteProperty("hive.tez.container.size", container_size)
+    putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", str(int(int(container_size)/3)*1024*1024))
     putHiveSiteProperty("hive.prewarm.enabled", "false")
     putHiveSiteProperty("hive.prewarm.numcontainers", "3")
     putHiveSiteProperty("hive.tez.auto.reducer.parallelism", "true")

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -874,7 +874,7 @@ class TestHDP22StackAdvisor(TestCase):
       },
       'hive-site': {
         'properties': {
-          'hive.auto.convert.join.noconditionaltask.size': '2147483648',
+          'hive.auto.convert.join.noconditionaltask.size': '89128960',
           'hive.cbo.enable': 'true',
           'hive.compactor.initiator.on': 'false',
           'hive.compactor.worker.threads': '0',