Browse Source

AMBARI-11014. HBase theme should use configs 'hbase.regionserver.global.memstore.size' and 'hbase.hstore.compaction.max' (srimanth)

Srimanth Gunturi 10 years ago
parent
commit
cf62f66a1c

+ 56 - 0
ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/hbase-site.xml

@@ -147,4 +147,60 @@
       </property>
       </property>
     </depends-on>
     </depends-on>
   </property>
   </property>
+  <property>
+    <name>hbase.hstore.compaction.max</name>
+    <value>10</value>
+    <description>The maximum number of StoreFiles which will be selected for a single minor
+      compaction, regardless of the number of eligible StoreFiles. Effectively, the value of
+      hbase.hstore.compaction.max controls the length of time it takes a single compaction to
+      complete. Setting it larger means that more StoreFiles are included in a compaction. For most
+      cases, the default value is appropriate.
+    </description>
+    <display-name>Maximum Files in a Store before Compaction</display-name>
+    <value-attributes>
+      <type>int</type>
+      <entries>
+        <entry>
+          <value>8</value>
+        </entry>
+        <entry>
+          <value>9</value>
+        </entry>
+        <entry>
+          <value>10</value>
+        </entry>
+        <entry>
+          <value>11</value>
+        </entry>
+        <entry>
+          <value>12</value>
+        </entry>
+        <entry>
+          <value>13</value>
+        </entry>
+        <entry>
+          <value>14</value>
+        </entry>
+        <entry>
+          <value>15</value>
+        </entry>
+      </entries>
+    </value-attributes>
+  </property>
+  <property>
+    <name>hbase.regionserver.global.memstore.size</name>
+    <value>0.4</value>
+    <description>Percentage of RegionServer memory to allocate to write buffers.
+      Each column family within each region is allocated a smaller pool (the memstore) within this shared write pool.
+      If this buffer is full, updates are blocked and data is flushed from memstores until a global low watermark
+      (hbase.regionserver.global.memstore.size.lower.limit) is reached.
+    </description>
+    <display-name>% of RegionServer Allocated to Write Buffers</display-name>
+    <value-attributes>
+      <type>float</type>
+      <minimum>0</minimum>
+      <maximum>0.8</maximum>
+      <increment-step>0.01</increment-step>
+    </value-attributes>
+  </property>
 </configuration>
 </configuration>

+ 4 - 4
ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/themes/theme.json

@@ -170,7 +170,7 @@
           "subsection-name": "subsection-hbase-memory-col1"
           "subsection-name": "subsection-hbase-memory-col1"
         },
         },
         {
         {
-          "config": "hbase-site/hbase.regionserver.global.memstore.upperLimit",
+          "config": "hbase-site/hbase.regionserver.global.memstore.size",
           "subsection-name": "subsection-hbase-memory-col1"
           "subsection-name": "subsection-hbase-memory-col1"
         },
         },
         {
         {
@@ -202,7 +202,7 @@
           "subsection-name": "subsection-hbase-disk-col2"
           "subsection-name": "subsection-hbase-disk-col2"
         },
         },
         {
         {
-          "config": "hbase-site/hbase.hstore.compactionThreshold",
+          "config": "hbase-site/hbase.hstore.compaction.max",
           "subsection-name": "subsection-hbase-disk-col3"
           "subsection-name": "subsection-hbase-disk-col3"
         },
         },
         {
         {
@@ -266,7 +266,7 @@
         }
         }
       },
       },
       {
       {
-        "config": "hbase-site/hbase.regionserver.global.memstore.upperLimit",
+        "config": "hbase-site/hbase.regionserver.global.memstore.size",
         "widget": {
         "widget": {
           "type": "slider",
           "type": "slider",
           "units": [
           "units": [
@@ -349,7 +349,7 @@
         }
         }
       },
       },
       {
       {
-        "config": "hbase-site/hbase.hstore.compactionThreshold",
+        "config": "hbase-site/hbase.hstore.compaction.max",
         "widget": {
         "widget": {
           "type": "combo"
           "type": "combo"
         }
         }

+ 3 - 3
ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -428,7 +428,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
 
 
     putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services)
     putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services)
     putHbaseSitePropertyAttributes = self.putPropertyAttribute(configurations, "hbase-site")
     putHbaseSitePropertyAttributes = self.putPropertyAttribute(configurations, "hbase-site")
-    putHbaseSiteProperty("hbase.regionserver.global.memstore.upperLimit", '0.4')
+    putHbaseSiteProperty("hbase.regionserver.global.memstore.size", '0.4')
 
 
     if 'hbase-env' in services['configurations'] and 'phoenix_sql_enabled' in services['configurations']['hbase-env']['properties']:
     if 'hbase-env' in services['configurations'] and 'phoenix_sql_enabled' in services['configurations']['hbase-env']['properties']:
       if 'true' == services['configurations']['hbase-env']['properties']['phoenix_sql_enabled'].lower():
       if 'true' == services['configurations']['hbase-env']['properties']['phoenix_sql_enabled'].lower():
@@ -468,7 +468,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       # Set values in hbase-site
       # Set values in hbase-site
       putHbaseProperty = self.putProperty(configurations, "hbase-site", services)
       putHbaseProperty = self.putProperty(configurations, "hbase-site", services)
       putHbaseProperty('hfile.block.cache.size', hfile_block_cache_size)
       putHbaseProperty('hfile.block.cache.size', hfile_block_cache_size)
-      putHbaseProperty('hbase.regionserver.global.memstore.upperLimit', hbase_regionserver_global_memstore_size)
+      putHbaseProperty('hbase.regionserver.global.memstore.size', hbase_regionserver_global_memstore_size)
       putHbaseProperty('hbase.bucketcache.ioengine', 'offheap')
       putHbaseProperty('hbase.bucketcache.ioengine', 'offheap')
       putHbaseProperty('hbase.bucketcache.size', hbase_bucketcache_size)
       putHbaseProperty('hbase.bucketcache.size', hbase_bucketcache_size)
       putHbaseProperty('hbase.bucketcache.percentage.in.combinedcache', hbase_bucketcache_percentage_in_combinedcache_str)
       putHbaseProperty('hbase.bucketcache.percentage.in.combinedcache', hbase_bucketcache_percentage_in_combinedcache_str)
@@ -823,7 +823,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     hbase_site = properties
     hbase_site = properties
     validationItems = []
     validationItems = []
 
 
-    prop_name1 = 'hbase.regionserver.global.memstore.upperLimit'
+    prop_name1 = 'hbase.regionserver.global.memstore.size'
     prop_name2 = 'hfile.block.cache.size'
     prop_name2 = 'hfile.block.cache.size'
     props_max_sum = 0.8
     props_max_sum = 0.8
 
 

+ 1 - 1
ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -1474,7 +1474,7 @@ class TestHDP22StackAdvisor(TestCase):
           "hbase.rpc.controllerfactory.class": "org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory",
           "hbase.rpc.controllerfactory.class": "org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory",
           "hbase.bucketcache.size": "",
           "hbase.bucketcache.size": "",
           "hbase.bucketcache.percentage.in.combinedcache": "",
           "hbase.bucketcache.percentage.in.combinedcache": "",
-          "hbase.regionserver.global.memstore.upperLimit": "0.4",
+          "hbase.regionserver.global.memstore.size": "0.4",
           "hbase.bucketcache.ioengine": ""
           "hbase.bucketcache.ioengine": ""
         },
         },
         'property_attributes': {
         'property_attributes': {