AMBARI-8054. Version comparison fails for GlusterFS (alejandro)

ambari-common/src/main/python/resource_management/libraries/functions/version.py

@@ -36,17 +36,19 @@ def _normalize(v, desired_segments=0):
 
 def format_hdp_stack_version(input):
   """
-  :param input: Input string, e.g. "2.2"
+  :param input: Input string, e.g. "2.2" or "GlusterFS"
   :return: Returns a well-formatted HDP stack version of the form #.#.#.# as a string.
   """
   if input:
-    normalized = _normalize(str(input))
-    if len(normalized) == 2:
-      normalized = normalized + [0, 0]
-    elif len(normalized) == 3:
-      normalized = normalized + [0, ]
-    normalized = [str(x) for x in normalized]   # need to convert each number into a string
-    return ".".join(normalized)
+    strip_dots = input.replace('.', '')
+    if strip_dots.isdigit():
+      normalized = _normalize(str(input))
+      if len(normalized) == 2:
+        normalized = normalized + [0, 0]
+      elif len(normalized) == 3:
+        normalized = normalized + [0, ]
+      normalized = [str(x) for x in normalized]   # need to convert each number into a string
+      return ".".join(normalized)
   return ""
 
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py

@@ -17,13 +17,15 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 from resource_management.core.system import System
 
 config = Script.get_config()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py

@@ -17,6 +17,7 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 import collections
 import json
@@ -33,7 +34,8 @@ java_home = config['hostLevelParams']['java_home']
 ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py

@@ -17,6 +17,7 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 from resource_management.core.system import System
 import json
@@ -26,7 +27,8 @@ config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #users and groups
 hbase_user = config['configurations']['hbase-env']['hbase_user']

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py

@@ -17,6 +17,7 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 from resource_management.core.system import System
 import os
@@ -24,7 +25,8 @@ import os
 config = Script.get_config()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/params.py

@@ -17,6 +17,7 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 
 config = Script.get_config()
@@ -27,7 +28,8 @@ proxyuser_group =  config['configurations']['hadoop-env']['proxyuser_group']
 security_enabled = False
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py

@@ -19,6 +19,7 @@ limitations under the License.
 """
 
 from functions import calc_xmn_from_xms
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 import status_params
 
@@ -27,7 +28,8 @@ config = Script.get_config()
 exec_tmp_dir = Script.get_tmp_dir()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py

@@ -17,6 +17,7 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 import status_params
 import os
@@ -27,7 +28,8 @@ config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py

@@ -18,8 +18,8 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
-from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
 import status_params
 import os
 
@@ -30,7 +30,7 @@ tmp_dir = Script.get_tmp_dir()
 # This is expected to be of the form #.#.#.#
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >=0
 
 # Hadoop params
 # TODO, this logic should initialize these parameters in a file inside the HDP 2.2 stack.

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py

@@ -18,6 +18,7 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 import status_params
 import os
@@ -27,7 +28,8 @@ config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = functions.version.compare_versions(hdp_stack_version, '2.1') >= 1
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py

@@ -19,6 +19,7 @@ Ambari Agent
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 
 # server configurations
@@ -26,7 +27,8 @@ config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/params.py

@@ -17,12 +17,14 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 
 config = Script.get_config()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py

@@ -19,20 +19,21 @@ Ambari Agent
 
 """
 import os
-
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
-from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
 import status_params
 
 # server configurations
 config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
+# This is expected to be of the form #.#.#.#
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
-if compare_versions(hdp_stack_version, "2.2.0.0") >= 0:
+if stack_is_hdp22_or_further:
   hadoop_libexec_dir = "/usr/hdp/current/hadoop-client/libexec"
   hadoop_bin = "/usr/hdp/current/hadoop-client/sbin"
   hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/ZOOKEEPER/package/scripts/params.py

@@ -19,6 +19,7 @@ Ambari Agent
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 import status_params
 
@@ -27,7 +28,8 @@ config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/package/scripts/params.py

@@ -17,6 +17,7 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 
 from status_params import *
@@ -24,7 +25,8 @@ from status_params import *
 config = Script.get_config()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/params.py

@@ -18,6 +18,7 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 import status_params
 
@@ -25,7 +26,8 @@ import status_params
 config = Script.get_config()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/params.py

@@ -18,8 +18,8 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
-from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
 
 # server configurations
 config = Script.get_config()
@@ -27,9 +27,9 @@ config = Script.get_config()
 # This is expected to be of the form #.#.#.#
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
-if compare_versions(hdp_stack_version, "2.2.0.0") >= 0:
+if stack_is_hdp22_or_further:
   hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
 else:
   hadoop_bin_dir = "/usr/bin"

ambari-server/src/main/resources/stacks/HDP/2.2/services/KAFKA/package/scripts/params.py

@@ -18,6 +18,7 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 import status_params
 
@@ -25,8 +26,8 @@ import status_params
 config = Script.get_config()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
-
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 if stack_is_hdp22_or_further:
     kafka_home = '/usr/hdp/current/kafka-broker/'

ambari-server/src/main/resources/stacks/HDP/2.2/services/KNOX/package/scripts/params.py

@@ -19,6 +19,7 @@ Ambari Agent
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 import status_params
 
@@ -26,7 +27,8 @@ config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 if stack_is_hdp22_or_further:
   knox_bin = '/usr/hdp/current/knox-server/bin/gateway.sh'

ambari-server/src/main/resources/stacks/HDP/2.2/services/SLIDER/package/scripts/params.py

@@ -18,13 +18,15 @@ limitations under the License.
 
 """
 
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 
 # server configurations
 config = Script.get_config()
 
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0
 
 #hadoop params
 if stack_is_hdp22_or_further:

ambari-server/src/test/python/TestVersion.py

@@ -37,11 +37,14 @@ class TestVersion(TestCase):
     l = [("2.2",   "2.2.0.0"),
          ("2.2.1", "2.2.1.0"),
          ("2.2.1.3", "2.2.1.3")]
-
+    
     for input, expected in l:
       actual = self.version_module.format_hdp_stack_version(input)
       self.assertEqual(expected, actual)
 
+    gluster_fs_actual = self.version_module.format_hdp_stack_version("GlusterFS")
+    self.assertEqual("", gluster_fs_actual)
+
   def test_comparison(self):
     # All versions to compare, from 1.0.0.0 to 3.0.0.0, and only include elements that are a multiple of 7.
     versions = range(1000, 3000, 7)
@@ -59,3 +62,10 @@ class TestVersion(TestCase):
     self.assertEqual(0, self.version_module.compare_versions("2.10", "2.10.0"))
     self.assertEqual(0, self.version_module.compare_versions("2.10", "2.10.0.0"))
     self.assertEqual(0, self.version_module.compare_versions("2.10.0", "2.10.0.0"))
+
+    try:
+      self.version_module.compare_versions("", "GlusterFS")
+    except ValueError:
+      pass
+    else:
+      self.fail("Did not raise exception")