AMBARI-11060 Pig-HA tests fails with java.lang.OutOfMemoryError: Java heap space (dsen)

ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -576,10 +576,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       nodemanagerMinRam = min(nodemanager["Hosts"]["total_mem"]/1024, nodemanagerMinRam)
     putMapredProperty('yarn.app.mapreduce.am.resource.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
     putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * int(configurations["mapred-site"]["properties"]["yarn.app.mapreduce.am.resource.mb"]))) + "m" + " -Dhdp.version=${hdp.version}")
-    putMapredProperty('mapreduce.map.memory.mb', int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
     putMapredProperty('mapreduce.reduce.memory.mb', min(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(nodemanagerMinRam)))
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    min_mapreduce_map_memory_mb = 0
+    min_mapreduce_map_java_opts = 0
+    if ("PIG" in servicesList):
+      min_mapreduce_map_memory_mb = 1500
+      min_mapreduce_map_java_opts = 1024
+    putMapredProperty('mapreduce.map.memory.mb', max(min_mapreduce_map_memory_mb, int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])))
     mapredMapXmx = int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]));
-    putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(mapredMapXmx) + "m")
+    putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(max(min_mapreduce_map_java_opts, mapredMapXmx)) + "m")
     putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"]))) + "m")
     putMapredProperty('mapreduce.task.io.sort.mb', str(min(int(0.7*mapredMapXmx), 2047)))
     # Property Attributes

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -1144,6 +1144,264 @@ class TestHDP22StackAdvisor(TestCase):
     self.assertEquals(configurations, expected)
 
 
+  def test_recommendMapredConfigurationAttributesWithPigService(self):
+    configurations = {
+      "mapred-site": {
+        "properties": {
+          "mapreduce.map.memory.mb": "1024",
+          "mapreduce.reduce.memory.mb": "682",
+          "yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}",
+          "mapreduce.reduce.java.opts": "-Xmx546m",
+          "yarn.app.mapreduce.am.resource.mb": "682",
+          "mapreduce.map.java.opts": "-Xmx546m",
+          "mapreduce.task.io.sort.mb": "273"
+        }
+      },
+      "yarn-site": {
+        "properties": {
+          "yarn.nodemanager.resource.memory-mb": "1280",
+          "yarn.scheduler.minimum-allocation-mb": "100",
+          "yarn.scheduler.maximum-allocation-mb": "1280",
+          "yarn.nodemanager.resource.cpu-vcores": "2"
+        },
+        }
+    }
+    clusterData = {
+      "cpu": 4,
+      "containers" : 5,
+      "ramPerContainer": 256
+    }
+    expected = {
+      "yarn-env": {
+        "properties": {
+          "min_user_id": "500"
+        }
+      },
+      "mapred-site": {
+        "properties": {
+          "mapreduce.map.memory.mb": "1500",
+          "mapreduce.reduce.memory.mb": "200",
+          "yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}",
+          "mapreduce.reduce.java.opts": "-Xmx160m",
+          "yarn.app.mapreduce.am.resource.mb": "100",
+          "mapreduce.map.java.opts": "-Xmx1200m",
+          "mapreduce.task.io.sort.mb": "840"
+        },
+        "property_attributes": {
+          'mapreduce.task.io.sort.mb': {'maximum': '2047'},
+          'yarn.app.mapreduce.am.resource.mb': {'maximum': '1280',
+                                                'minimum': '100'},
+          'mapreduce.map.memory.mb': {'maximum': '1280',
+                                      'minimum': '100'},
+          'mapreduce.reduce.memory.mb': {'maximum': '1280',
+                                         'minimum': '100'}
+        }
+      },
+      "yarn-site": {
+        "properties": {
+          "yarn.nodemanager.resource.memory-mb": "1280",
+          "yarn.scheduler.minimum-allocation-mb": "100",
+          "yarn.scheduler.maximum-allocation-vcores": "1",
+          "yarn.scheduler.minimum-allocation-vcores": "1",
+          "yarn.scheduler.maximum-allocation-mb": "1280",
+          "yarn.nodemanager.resource.cpu-vcores": "1"
+        },
+        "property_attributes": {
+          'yarn.nodemanager.resource.memory-mb': {'maximum': '1877'},
+          'yarn.nodemanager.resource.cpu-vcores': {'maximum': '2'},
+          'yarn.scheduler.minimum-allocation-vcores': {'maximum': '1'},
+          'yarn.scheduler.maximum-allocation-vcores': {'maximum': '1'},
+          'yarn.scheduler.minimum-allocation-mb': {'maximum': '1280'},
+          'yarn.scheduler.maximum-allocation-mb': {'maximum': '1280'}
+        }
+      }
+    }
+    services = {
+      "services": [
+        {
+          "href": "/api/v1/stacks/HDP/versions/2.2/services/PIG",
+          "StackServices": {
+            "service_name": "PIG",
+            "service_version": "2.6.0.2.2",
+            "stack_name": "HDP",
+            "stack_version": "2.2"
+          }, "components": [
+          {
+            "StackServiceComponents": {
+              "advertise_version": "true",
+              "cardinality": "0+",
+              "component_category": "CLIENT",
+              "component_name": "PIG",
+              "display_name": "Pig",
+              "is_client": "true",
+              "is_master": "false",
+              "hostnames": []
+            },
+            "dependencies": []
+          }
+        ]
+        },
+        {
+          "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2",
+          "StackServices" : {
+            "service_name" : "MAPREDUCE2",
+            "service_version" : "2.6.0.2.2",
+            "stack_name" : "HDP",
+            "stack_version" : "2.2"
+          },
+          "components" : [ {
+                             "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/HISTORYSERVER",
+                             "StackServiceComponents" : {
+                               "advertise_version" : "true",
+                               "cardinality" : "1",
+                               "component_category" : "MASTER",
+                               "component_name" : "HISTORYSERVER",
+                               "custom_commands" : [ ],
+                               "display_name" : "History Server",
+                               "is_client" : "false",
+                               "is_master" : "true",
+                               "service_name" : "MAPREDUCE2",
+                               "stack_name" : "HDP",
+                               "stack_version" : "2.2",
+                               "hostnames" : [ "c6402.ambari.apache.org" ]
+                             },
+                             "auto_deploy" : {
+                               "enabled" : "true",
+                               "location" : "YARN/RESOURCEMANAGER"
+                             },
+                             "dependencies" : [ {
+                                                  "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/HISTORYSERVER/dependencies/HDFS_CLIENT",
+                                                  "Dependencies" : {
+                                                    "component_name" : "HDFS_CLIENT",
+                                                    "dependent_component_name" : "HISTORYSERVER",
+                                                    "dependent_service_name" : "MAPREDUCE2",
+                                                    "stack_name" : "HDP",
+                                                    "stack_version" : "2.2"
+                                                  }
+                                                } ]
+                           }]},
+        {
+          "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN",
+          "StackServices": {
+            "service_name": "YARN",
+            "service_version": "2.6.0.2.2",
+            "stack_name": "HDP",
+            "stack_version": "2.2"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "advertise_version": "false",
+                "cardinality": "1",
+                "component_category": "MASTER",
+                "component_name": "APP_TIMELINE_SERVER",
+                "display_name": "App Timeline Server",
+                "is_client": "false",
+                "is_master": "true",
+                "hostnames": []
+              },
+              "dependencies": []
+            },
+            {
+              "StackServiceComponents": {
+                "advertise_version": "true",
+                "cardinality": "1+",
+                "component_category": "SLAVE",
+                "component_name": "NODEMANAGER",
+                "display_name": "NodeManager",
+                "is_client": "false",
+                "is_master": "false",
+                "hostnames": [
+                  "c6403.ambari.apache.org"
+                ]
+              },
+              "dependencies": []
+            },
+            {
+              "StackServiceComponents": {
+                "advertise_version": "true",
+                "cardinality": "1-2",
+                "component_category": "MASTER",
+                "component_name": "RESOURCEMANAGER",
+                "display_name": "ResourceManager",
+                "is_client": "false",
+                "is_master": "true",
+                "hostnames": []
+              },
+              "dependencies": []
+            },
+            {
+              "StackServiceComponents": {
+                "advertise_version": "true",
+                "cardinality": "1+",
+                "component_category": "CLIENT",
+                "component_name": "YARN_CLIENT",
+                "display_name": "YARN Client",
+                "is_client": "true",
+                "is_master": "false",
+                "hostnames": []
+              },
+              "dependencies": []
+            }
+          ]
+        },
+        ],
+      "configurations": configurations,
+      "changed-configurations": [
+        {
+          "type": "yarn-site",
+          "name": "yarn.scheduler.minimum-allocation-mb"
+        },
+        ]
+
+    }
+    hosts = {
+      "items" : [
+        {
+          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+          "Hosts" : {
+            "cpu_count" : 1,
+            "host_name" : "c6401.ambari.apache.org",
+            "os_arch" : "x86_64",
+            "os_type" : "centos6",
+            "ph_cpu_count" : 1,
+            "public_host_name" : "c6401.ambari.apache.org",
+            "rack_info" : "/default-rack",
+            "total_mem" : 1922680
+          }
+        },
+        {
+          "href" : "/api/v1/hosts/c6402.ambari.apache.org",
+          "Hosts" : {
+            "cpu_count" : 1,
+            "host_name" : "c6402.ambari.apache.org",
+            "os_arch" : "x86_64",
+            "os_type" : "centos6",
+            "ph_cpu_count" : 1,
+            "public_host_name" : "c6402.ambari.apache.org",
+            "rack_info" : "/default-rack",
+            "total_mem" : 1922680
+          }
+        },
+        {
+          "href" : "/api/v1/hosts/c6403.ambari.apache.org",
+          "Hosts" : {
+            "cpu_count" : 1,
+            "host_name" : "c6403.ambari.apache.org",
+            "os_arch" : "x86_64",
+            "os_type" : "centos6",
+            "ph_cpu_count" : 1,
+            "public_host_name" : "c6403.ambari.apache.org",
+            "rack_info" : "/default-rack",
+            "total_mem" : 1922680
+          }
+        }
+      ]
+    }
+
+    self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts)
+    self.assertEquals(configurations, expected)
+
   def test_recommendMapredConfigurationAttributes(self):
     configurations = {
       "mapred-site": {