AMBARI-12329 - Review HIVE recommendations and changes on upgrade

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java

@@ -1560,21 +1560,51 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
         for (final Cluster cluster : clusterMap.values()) {
           String content = null;
           if(cluster.getDesiredConfigByType("hive-env") != null) {
-            Map<String, String> hiveProps = new HashMap<String, String>();
+            Map<String, String> hiveEnvProps = new HashMap<String, String>();
+            Set<String> hiveServerSiteRemoveProps = new HashSet<String>();
             // Update logic for setting HIVE_AUX_JARS_PATH in hive-env.sh
             content = cluster.getDesiredConfigByType("hive-env").getProperties().get("content");
             if(content != null) {
               content = updateHiveEnvContent(content);
-              hiveProps.put("content", content);
+              hiveEnvProps.put("content", content);
             }
             //hive metastore and client_heapsize are added for HDP2, we should check if it exists and not add it for HDP1
             if (!cluster.getDesiredConfigByType("hive-env").getProperties().containsKey("hive.client.heapsize")) {
-              hiveProps.put("hive.client.heapsize", "512m");
+              hiveEnvProps.put("hive.client.heapsize", "512m");
             }
             if (!cluster.getDesiredConfigByType("hive-env").getProperties().containsKey("hive.metastore.heapsize")) {
-              hiveProps.put("hive.metastore.heapsize", "1024m");
+              hiveEnvProps.put("hive.metastore.heapsize", "1024m");
             }
-            updateConfigurationPropertiesForCluster(cluster, "hive-env", hiveProps, true, true);
+            if (cluster.getDesiredConfigByType("hive-env").getProperties().containsKey("hive_security_authorization") &&
+                    "none".equalsIgnoreCase(cluster.getDesiredConfigByType("hive-env").getProperties().get("hive_security_authorization"))) {
+              hiveServerSiteRemoveProps.add("hive.security.authorization.manager");
+              hiveServerSiteRemoveProps.add("hive.security.authenticator.manager");
+            }
+            updateConfigurationPropertiesForCluster(cluster, "hive-env", hiveEnvProps, true, true);
+            updateConfigurationPropertiesForCluster(cluster, "hiveserver2-site", new HashMap<String, String>(), hiveServerSiteRemoveProps, false, true);
+          }
+
+          if(cluster.getDesiredConfigByType("hive-site") != null) {
+            Set<String> hiveSiteRemoveProps = new HashSet<String>();
+            String hive_server2_auth = "";
+            if (cluster.getDesiredConfigByType("hive-site").getProperties().containsKey("hive.server2.authentication")) {
+              hive_server2_auth = cluster.getDesiredConfigByType("hive-site").getProperties().get("hive.server2.authentication");
+            }
+            if (!"pam".equalsIgnoreCase(hive_server2_auth)) {
+              hiveSiteRemoveProps.add("hive.server2.authentication.pam.services");
+            }
+            if (!"custom".equalsIgnoreCase(hive_server2_auth)) {
+              hiveSiteRemoveProps.add("hive.server2.custom.authentication.class");
+            }
+            if (!"ldap".equalsIgnoreCase(hive_server2_auth)) {
+              hiveSiteRemoveProps.add("hive.server2.authentication.ldap.url");
+              hiveSiteRemoveProps.add("hive.server2.authentication.ldap.baseDN");
+            }
+            if (!"kerberos".equalsIgnoreCase(hive_server2_auth) && !cluster.getServices().containsKey("KERBEROS")) {
+              hiveSiteRemoveProps.add("hive.server2.authentication.kerberos.keytab");
+              hiveSiteRemoveProps.add("hive.server2.authentication.kerberos.principal");
+            }
+            updateConfigurationPropertiesForCluster(cluster, "hive-site", new HashMap<String, String>(), hiveSiteRemoveProps, false, true);
           }
         }
       }

ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -379,8 +379,14 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     # this property is unrelated to Kerberos
     if str(configurations["hive-env"]["properties"]["hive_security_authorization"]).lower() == "none":
       putHiveSiteProperty("hive.security.authorization.manager", "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory")
-      putHiveServerPropertyAttribute("hive.security.authorization.manager", "delete", "true")
-      putHiveServerPropertyAttribute("hive.security.authenticator.manager", "delete", "true")
+      if ("hive.security.authorization.manager" in configurations["hiveserver2-site"]["properties"]) or \
+              ("hiveserver2-site" not in services["configurations"]) or \
+              ("hiveserver2-site" in services["configurations"] and "hive.security.authorization.manager" in services["configurations"]["hiveserver2-site"]["properties"]):
+        putHiveServerPropertyAttribute("hive.security.authorization.manager", "delete", "true")
+      if ("hive.security.authenticator.manager" in configurations["hiveserver2-site"]["properties"]) or \
+              ("hiveserver2-site" not in services["configurations"]) or \
+              ("hiveserver2-site" in services["configurations"] and "hive.security.authenticator.manager" in services["configurations"]["hiveserver2-site"]["properties"]):
+        putHiveServerPropertyAttribute("hive.security.authenticator.manager", "delete", "true")
       if "KERBEROS" not in servicesList: # Kerberos security depends on this property
         putHiveSiteProperty("hive.security.authorization.enabled", "false")
     else:
@@ -431,25 +437,43 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       putHiveSiteProperty("hive.server2.authentication.ldap.url", "")
       putHiveSiteProperty("hive.server2.authentication.ldap.baseDN", " ")
     else:
-      putHiveSitePropertyAttribute("hive.server2.authentication.ldap.url", "delete", "true")
-      putHiveSitePropertyAttribute("hive.server2.authentication.ldap.baseDN", "delete", "true")
+      if ("hive.server2.authentication.ldap.url" in configurations["hive-site"]["properties"]) or \
+              ("hive-site" not in services["configurations"]) or \
+              ("hive-site" in services["configurations"] and "hive.server2.authentication.ldap.url" in services["configurations"]["hive-site"]["properties"]):
+        putHiveSitePropertyAttribute("hive.server2.authentication.ldap.url", "delete", "true")
+      if ("hive.server2.authentication.ldap.baseDN" in configurations["hive-site"]["properties"]) or \
+              ("hive-site" not in services["configurations"]) or \
+              ("hive-site" in services["configurations"] and "hive.server2.authentication.ldap.baseDN" in services["configurations"]["hive-site"]["properties"]):
+        putHiveSitePropertyAttribute("hive.server2.authentication.ldap.baseDN", "delete", "true")
 
     if hive_server2_auth == "kerberos":
       putHiveSiteProperty("hive.server2.authentication.kerberos.keytab", "")
       putHiveSiteProperty("hive.server2.authentication.kerberos.principal", "")
     elif "KERBEROS" not in servicesList: # Since 'hive_server2_auth' cannot be relied on within the default, empty recommendations request
-      putHiveSitePropertyAttribute("hive.server2.authentication.kerberos.keytab", "delete", "true")
-      putHiveSitePropertyAttribute("hive.server2.authentication.kerberos.principal", "delete", "true")
+      if ("hive.server2.authentication.kerberos.keytab" in configurations["hive-site"]["properties"]) or \
+              ("hive-site" not in services["configurations"]) or \
+              ("hive-site" in services["configurations"] and "hive.server2.authentication.kerberos.keytab" in services["configurations"]["hive-site"]["properties"]):
+        putHiveSitePropertyAttribute("hive.server2.authentication.kerberos.keytab", "delete", "true")
+      if ("hive.server2.authentication.kerberos.principal" in configurations["hive-site"]["properties"]) or \
+              ("hive-site" not in services["configurations"]) or \
+              ("hive-site" in services["configurations"] and "hive.server2.authentication.kerberos.principal" in services["configurations"]["hive-site"]["properties"]):
+        putHiveSitePropertyAttribute("hive.server2.authentication.kerberos.principal", "delete", "true")
 
     if hive_server2_auth == "pam":
       putHiveSiteProperty("hive.server2.authentication.pam.services", "")
     else:
-      putHiveSitePropertyAttribute("hive.server2.authentication.pam.services", "delete", "true")
+      if ("hive.server2.authentication.pam.services" in configurations["hive-site"]["properties"]) or \
+              ("hive-site" not in services["configurations"]) or \
+              ("hive-site" in services["configurations"] and "hive.server2.authentication.pam.services" in services["configurations"]["hive-site"]["properties"]):
+        putHiveSitePropertyAttribute("hive.server2.authentication.pam.services", "delete", "true")
 
     if hive_server2_auth == "custom":
       putHiveSiteProperty("hive.server2.custom.authentication.class", "")
     else:
-      putHiveSitePropertyAttribute("hive.server2.custom.authentication.class", "delete", "true")
+      if ("hive.server2.authentication" in configurations["hive-site"]["properties"]) or \
+              ("hive-site" not in services["configurations"]) or \
+              ("hive-site" in services["configurations"] and "hive.server2.custom.authentication.class" in services["configurations"]["hive-site"]["properties"]):
+        putHiveSitePropertyAttribute("hive.server2.custom.authentication.class", "delete", "true")
 
 
   def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java

@@ -316,9 +316,24 @@ public class UpgradeCatalog210Test {
 
     final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class);
     final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class);
+    final ServiceConfigVersionResponse mockServiceConfigVersionResponse = easyMockSupport.createNiceMock(ServiceConfigVersionResponse.class);
     final Config mockHiveEnv = easyMockSupport.createNiceMock(Config.class);
+    final Config mockHiveServerSite = easyMockSupport.createNiceMock(Config.class);
+    final Config mockHiveSite = easyMockSupport.createNiceMock(Config.class);
+
+    final Map<String, String> propertiesExpectedHiveEnv = new HashMap<String, String>() {{
+      put("hive_security_authorization", "none");
+    }};
+    final Map<String, String> propertiesExpectedHiveSite = new HashMap<String, String>() {{
+      put("hive.server2.authentication", "pam");
+      put("hive.server2.custom.authentication.class", "");
+    }};
+    final Map<String, String> propertiesExpectedHiveServerSite = new HashMap<String, String>() {{
+      put("hive.security.authorization.manager", "");
+      put("hive.security.authenticator.manager", "");
+    }};
+    final Map<String, Service> servicesExpected = new HashMap<String, Service>();
 
-    final Map<String, String> propertiesExpectedHiveEnv = new HashMap<String, String>();
     final Injector mockInjector = Guice.createInjector(new AbstractModule() {
       @Override
       protected void configure() {
@@ -336,17 +351,32 @@ public class UpgradeCatalog210Test {
       put("normal", mockClusterExpected);
     }}).once();
 
+    Capture<String> configTypeEnv = new Capture<String>();
+    Capture<String> configTypeSite = new Capture<String>();
+    Capture<String> configTypeServerSite = new Capture<String>();
+
     expect(mockClusterExpected.getDesiredConfigByType("hive-env")).andReturn(mockHiveEnv).atLeastOnce();
+    expect(mockClusterExpected.getDesiredConfigByType("hiveserver2-site")).andReturn(mockHiveServerSite).atLeastOnce();
     expect(mockHiveEnv.getProperties()).andReturn(propertiesExpectedHiveEnv).anyTimes();
-    expect(mockClusterExpected.getConfig(anyObject(String.class), anyObject(String.class))).
-        andReturn(mockHiveEnv).anyTimes();
-    ServiceConfigVersionResponse r = null;
-    expect(mockClusterExpected.addDesiredConfig("ambari-upgrade", Collections.singleton(mockHiveEnv))).
-        andReturn(r).once();
+    expect(mockHiveServerSite.getProperties()).andReturn(propertiesExpectedHiveServerSite).anyTimes();
+    expect(mockClusterExpected.getConfig(capture(configTypeEnv), anyObject(String.class))).andReturn(mockHiveEnv).once();
+    expect(mockClusterExpected.getConfig(capture(configTypeServerSite), anyObject(String.class))).andReturn(mockHiveServerSite).once();
+    expect(mockClusterExpected.addDesiredConfig("ambari-upgrade", Collections.singleton(mockHiveEnv))).andReturn(mockServiceConfigVersionResponse).once();
+    expect(mockClusterExpected.addDesiredConfig("ambari-upgrade", Collections.singleton(mockHiveServerSite))).andReturn(mockServiceConfigVersionResponse).once();
+
+    expect(mockClusterExpected.getDesiredConfigByType("hive-site")).andReturn(mockHiveSite).atLeastOnce();
+    expect(mockHiveSite.getProperties()).andReturn(propertiesExpectedHiveSite).anyTimes();
+    expect(mockClusterExpected.getServices()).andReturn(servicesExpected).once();
+    expect(mockClusterExpected.getConfig(capture(configTypeSite), anyObject(String.class))).andReturn(mockHiveSite).once();
+    expect(mockClusterExpected.addDesiredConfig("ambari-upgrade", Collections.singleton(mockHiveSite))).andReturn(mockServiceConfigVersionResponse).once();
 
     easyMockSupport.replayAll();
     mockInjector.getInstance(UpgradeCatalog210.class).updateHiveConfigs();
     easyMockSupport.verifyAll();
+
+    assertEquals("hive-env", configTypeEnv.getValue());
+    assertEquals("hive-site", configTypeSite.getValue());
+    assertEquals("hiveserver2-site", configTypeServerSite.getValue());
   }
 
   @Test

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -953,20 +953,11 @@ class TestHDP22StackAdvisor(TestCase):
       "yarn-site": {
         "properties": {
           "yarn.scheduler.minimum-allocation-mb": "256",
-          "yarn.scheduler.maximum-allocation-mb": "8192",
-        },
-      },
-      "capacity-scheduler": {
-        "properties": {
-          "yarn.scheduler.capacity.root.queues": "queue1,queue2"
-        }
-      },
-      "hive-site": {
-        "properties": {
-          "hive.server2.authentication": "none"
+          "yarn.scheduler.maximum-allocation-mb": "8192"
         }
       }
     }
+
     clusterData = {
       "cpu": 4,
       "mapMemory": 3000,
@@ -975,12 +966,8 @@ class TestHDP22StackAdvisor(TestCase):
       "containers": 3,
       "ramPerContainer": 256
     }
+
     expected = {
-      'capacity-scheduler': {
-        'properties': {
-          'yarn.scheduler.capacity.root.queues': 'queue1,queue2'
-        }
-      },
       'yarn-site': {
         'properties': {
           'yarn.scheduler.minimum-allocation-mb': '256',
@@ -1035,8 +1022,7 @@ class TestHDP22StackAdvisor(TestCase):
           'hive.vectorized.execution.enabled': 'true',
           'hive.vectorized.execution.reduce.enabled': 'false',
           'hive.security.metastore.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider',
-          'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory',
-          "hive.server2.authentication": "none"
+          'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory'
         },
        'property_attributes': {
          'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'},
@@ -1060,6 +1046,7 @@ class TestHDP22StackAdvisor(TestCase):
         }
       }
     }
+
     services = {
       "services": [
         {
@@ -1128,10 +1115,37 @@ class TestHDP22StackAdvisor(TestCase):
           ]
         },
       ],
-      "configurations": configurations,
+      "configurations": {
+        "capacity-scheduler": {
+          "properties": {
+            "yarn.scheduler.capacity.root.queues": "queue1,queue2"
+          }
+        },
+        "hive-env": {
+          "properties": {
+          }
+        },
+        "hive-site": {
+          "properties": {
+            "hive.server2.authentication": "none",
+            "hive.server2.authentication.ldap.url": "",
+            "hive.server2.authentication.ldap.baseDN": "",
+            "hive.server2.authentication.kerberos.keytab": "",
+            "hive.server2.authentication.kerberos.principal": "",
+            "hive.server2.authentication.pam.services": "",
+            "hive.server2.custom.authentication.class": ""
+          }
+        },
+        "hiveserver2-site": {
+          "properties": {
+            "hive.security.authorization.manager": "",
+            "hive.security.authenticator.manager": ""
+          }
+        }
+      },
       "changed-configurations": [ ]
-
     }
+
     hosts = {
       "items" : [
         {
@@ -1180,8 +1194,8 @@ class TestHDP22StackAdvisor(TestCase):
     self.assertEquals(configurations, expected)
 
     #test recommendations
-    configurations["hive-site"]["properties"]["hive.cbo.enable"] = "false"
-    configurations["hive-env"]["properties"]["hive_security_authorization"] = "sqlstdauth"
+    services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"] = "false"
+    services["configurations"]["hive-env"]["properties"]["hive_security_authorization"] = "sqlstdauth"
     services["changed-configurations"] = [{"type": "hive-site", "name": "hive.cbo.enable"},
                                           {"type": "hive-env", "name": "hive_security_authorization"}]
     expected["hive-env"]["properties"]["hive_security_authorization"] = "sqlstdauth"
@@ -1201,7 +1215,7 @@ class TestHDP22StackAdvisor(TestCase):
 
 
     # test 'hive_security_authorization'=='sqlstdauth' => 'hive.server2.enable.doAs'=='false'
-    configurations["hive-env"]["properties"]["hive_security_authorization"] = "none"
+    services["configurations"]["hive-env"]["properties"]["hive_security_authorization"] = "none"
     expected["hive-env"]["properties"]["hive_security_authorization"] = "none"
     expected["hive-site"]["properties"]["hive.security.authorization.enabled"]="false"
     expected["hive-site"]["properties"]["hive.server2.enable.doAs"]="true"
@@ -1211,7 +1225,7 @@ class TestHDP22StackAdvisor(TestCase):
     self.assertEquals(configurations, expected)
 
     # test 'hive.server2.tez.default.queues' leaf queues
-    configurations['capacity-scheduler']['properties'] = {
+    services["configurations"]['capacity-scheduler']['properties'] = {
             "yarn.scheduler.capacity.maximum-am-resource-percent": "0.2",
             "yarn.scheduler.capacity.maximum-applications": "10000",
             "yarn.scheduler.capacity.node-locality-delay": "40",

ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py

@@ -254,11 +254,6 @@ class TestHDP23StackAdvisor(TestCase):
           "yarn.scheduler.minimum-allocation-mb": "256",
           "yarn.scheduler.maximum-allocation-mb": "8192",
         },
-      },
-      "capacity-scheduler": {
-        "properties": {
-          "yarn.scheduler.capacity.root.queues": "queue1,queue2"
-        }
       }
     }
     clusterData = {
@@ -270,11 +265,6 @@ class TestHDP23StackAdvisor(TestCase):
       "ramPerContainer": 256
     }
     expected = {
-      'capacity-scheduler': {
-        'properties': {
-          'yarn.scheduler.capacity.root.queues': 'queue1,queue2'
-        }
-      },
       'yarn-site': {
         'properties': {
           'yarn.scheduler.minimum-allocation-mb': '256',
@@ -421,7 +411,34 @@ class TestHDP23StackAdvisor(TestCase):
           ]
         },
       ],
-      "configurations": configurations,
+      "configurations": {
+        "capacity-scheduler": {
+          "properties": {
+            "yarn.scheduler.capacity.root.queues": "queue1,queue2"
+          }
+        },
+        "hive-env": {
+          "properties": {
+          }
+        },
+        "hive-site": {
+          "properties": {
+            "hive.server2.authentication": "none",
+            "hive.server2.authentication.ldap.url": "",
+            "hive.server2.authentication.ldap.baseDN": "",
+            "hive.server2.authentication.kerberos.keytab": "",
+            "hive.server2.authentication.kerberos.principal": "",
+            "hive.server2.authentication.pam.services": "",
+            "hive.server2.custom.authentication.class": ""
+          }
+        },
+        "hiveserver2-site": {
+          "properties": {
+            "hive.security.authorization.manager": "",
+            "hive.security.authenticator.manager": ""
+          }
+        }
+      },
       "changed-configurations": [ ]
 
     }