AMBARI-11942. YARN Queues not showing up in Hive configs (srimanth)

ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml

@@ -1592,6 +1592,12 @@ limitations under the License.
       </entries>
       <selection-cardinality>1+</selection-cardinality>
     </value-attributes>
+    <depends-on>
+      <property>
+        <type>capacity-scheduler</type>
+        <name>yarn.scheduler.capacity.root.queues</name>
+      </property>
+    </depends-on>
   </property>
 
   <property>

ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -348,17 +348,27 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     putHiveSiteProperty("hive.server2.enable.doAs", "true")
 
     yarn_queues = "default"
-    if "capacity-scheduler" in configurations and \
-      "yarn.scheduler.capacity.root.queues" in configurations["capacity-scheduler"]["properties"]:
-      yarn_queues = str(configurations["capacity-scheduler"]["properties"]["yarn.scheduler.capacity.root.queues"])
-    putHiveSiteProperty("hive.server2.tez.default.queues", yarn_queues)
-
+    capacitySchedulerProperties = services['configurations']["capacity-scheduler"]["properties"] if "capacity-scheduler" in services['configurations'] else {}
+    if "yarn.scheduler.capacity.root.queues" in capacitySchedulerProperties:
+      yarn_queues = str(capacitySchedulerProperties["yarn.scheduler.capacity.root.queues"])
     # Interactive Queues property attributes
     putHiveServerPropertyAttribute = self.putPropertyAttribute(configurations, "hiveserver2-site")
-    entries = []
-    for queue in yarn_queues.split(","):
-      entries.append({"label": str(queue) + " queue", "value": queue})
-    putHiveSitePropertyAttribute("hive.server2.tez.default.queues", "entries", entries)
+    toProcessQueues = yarn_queues.split(",")
+    leafQueues = []
+    while len(toProcessQueues) > 0:
+      queue = toProcessQueues.pop();
+      queueKey = "yarn.scheduler.capacity.root." + queue + ".queues"
+      if queueKey in capacitySchedulerProperties:
+        # This is a parent queue - need to add children
+        subQueues = capacitySchedulerProperties[queueKey].split(",")
+        for subQueue in subQueues:
+          toProcessQueues.append(queue + "." + subQueue)
+      else:
+        # This is a leaf queue
+        leafQueues.append({"label": str(queue) + " queue", "value": queue})
+    leafQueues = sorted(leafQueues, key=lambda q:q['value'])
+    putHiveSitePropertyAttribute("hive.server2.tez.default.queues", "entries", leafQueues)
+    putHiveSiteProperty("hive.server2.tez.default.queues", ",".join([leafQueue['value'] for leafQueue in leafQueues]))
 
     # Security
     putHiveEnvProperty("hive_security_authorization", "None")

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -1181,6 +1181,65 @@ class TestHDP22StackAdvisor(TestCase):
     self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
+    # test 'hive.server2.tez.default.queues' leaf queues
+    configurations['capacity-scheduler']['properties'] = {
+            "yarn.scheduler.capacity.maximum-am-resource-percent": "0.2",
+            "yarn.scheduler.capacity.maximum-applications": "10000",
+            "yarn.scheduler.capacity.node-locality-delay": "40",
+            "yarn.scheduler.capacity.queue-mappings-override.enable": "false",
+            "yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator",
+            "yarn.scheduler.capacity.root.accessible-node-labels": "*",
+            "yarn.scheduler.capacity.root.acl_administer_queue": "*",
+            "yarn.scheduler.capacity.root.capacity": "100",
+            "yarn.scheduler.capacity.root.default.a.a1.acl_administer_queue": "*",
+            "yarn.scheduler.capacity.root.default.a.a1.acl_submit_applications": "*",
+            "yarn.scheduler.capacity.root.default.a.a1.capacity": "75",
+            "yarn.scheduler.capacity.root.default.a.a1.maximum-capacity": "100",
+            "yarn.scheduler.capacity.root.default.a.a1.minimum-user-limit-percent": "100",
+            "yarn.scheduler.capacity.root.default.a.a1.ordering-policy": "fifo",
+            "yarn.scheduler.capacity.root.default.a.a1.state": "RUNNING",
+            "yarn.scheduler.capacity.root.default.a.a1.user-limit-factor": "1",
+            "yarn.scheduler.capacity.root.default.a.a2.acl_administer_queue": "*",
+            "yarn.scheduler.capacity.root.default.a.a2.acl_submit_applications": "*",
+            "yarn.scheduler.capacity.root.default.a.a2.capacity": "25",
+            "yarn.scheduler.capacity.root.default.a.a2.maximum-capacity": "25",
+            "yarn.scheduler.capacity.root.default.a.a2.minimum-user-limit-percent": "100",
+            "yarn.scheduler.capacity.root.default.a.a2.ordering-policy": "fifo",
+            "yarn.scheduler.capacity.root.default.a.a2.state": "RUNNING",
+            "yarn.scheduler.capacity.root.default.a.a2.user-limit-factor": "1",
+            "yarn.scheduler.capacity.root.default.a.acl_administer_queue": "*",
+            "yarn.scheduler.capacity.root.default.a.acl_submit_applications": "*",
+            "yarn.scheduler.capacity.root.default.a.capacity": "50",
+            "yarn.scheduler.capacity.root.default.a.maximum-capacity": "100",
+            "yarn.scheduler.capacity.root.default.a.minimum-user-limit-percent": "100",
+            "yarn.scheduler.capacity.root.default.a.ordering-policy": "fifo",
+            "yarn.scheduler.capacity.root.default.a.queues": "a1,a2",
+            "yarn.scheduler.capacity.root.default.a.state": "RUNNING",
+            "yarn.scheduler.capacity.root.default.a.user-limit-factor": "1",
+            "yarn.scheduler.capacity.root.default.acl_submit_applications": "*",
+            "yarn.scheduler.capacity.root.default.b.acl_administer_queue": "*",
+            "yarn.scheduler.capacity.root.default.b.acl_submit_applications": "*",
+            "yarn.scheduler.capacity.root.default.b.capacity": "50",
+            "yarn.scheduler.capacity.root.default.b.maximum-capacity": "50",
+            "yarn.scheduler.capacity.root.default.b.minimum-user-limit-percent": "100",
+            "yarn.scheduler.capacity.root.default.b.ordering-policy": "fifo",
+            "yarn.scheduler.capacity.root.default.b.state": "RUNNING",
+            "yarn.scheduler.capacity.root.default.b.user-limit-factor": "1",
+            "yarn.scheduler.capacity.root.default.capacity": "100",
+            "yarn.scheduler.capacity.root.default.maximum-capacity": "100",
+            "yarn.scheduler.capacity.root.default.queues": "a,b",
+            "yarn.scheduler.capacity.root.default.state": "RUNNING",
+            "yarn.scheduler.capacity.root.default.user-limit-factor": "1",
+            "yarn.scheduler.capacity.root.queues": "default"
+          }
+    expected['hive-site']['properties']['hive.server2.tez.default.queues'] = 'default.a.a1,default.a.a2,default.b'
+    expected['hive-site']['property_attributes']['hive.server2.tez.default.queues'] = {
+           'entries': [{'value': 'default.a.a1', 'label': 'default.a.a1 queue'}, {'value': 'default.a.a2', 'label': 'default.a.a2 queue'}, {'value': 'default.b', 'label': 'default.b queue'}]
+          }
+    self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations['hive-site']['property_attributes']['hive.server2.tez.default.queues'], expected['hive-site']['property_attributes']['hive.server2.tez.default.queues'])
+    self.assertEquals(configurations['hive-site']['properties']['hive.server2.tez.default.queues'], expected['hive-site']['properties']['hive.server2.tez.default.queues'])
+
   def test_recommendMapredConfigurationAttributesWithPigService(self):
     configurations = {
       "mapred-site": {