AMBARI-11526. White Label Stack Definition (echekanskiy via dlysnichenko)

ambari-common/src/main/python/white_labeling/WhiteLabelStackDefinition.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+import getopt
+import json
+import os
+import shutil
+import xml.etree.ElementTree as ET
+import re
+
+
+class _named_dict(dict):
+  """
+  Allow to get dict items using attribute notation, eg dict.attr == dict['attr']
+  """
+
+  def __init__(self, _dict):
+
+    def repl_list(_list):
+      for i, e in enumerate(_list):
+        if isinstance(e, list):
+          _list[i] = repl_list(e)
+        if isinstance(e, dict):
+          _list[i] = _named_dict(e)
+      return _list
+
+    dict.__init__(self, _dict)
+    for key, value in self.iteritems():
+      if isinstance(value, dict):
+        self[key] = _named_dict(value)
+      if isinstance(value, list):
+        self[key] = repl_list(value)
+
+  def __getattr__(self, item):
+    if item in self:
+      return self[item]
+    else:
+      dict.__getattr__(self, item)
+
+
+def copy_tree(src, dest, exclude=None, post_copy=None):
+  """
+  Copy files form src to dest.
+
+  :param src: source folder
+  :param dest: destination folder
+  :param exclude: list for excluding, eg [".xml"] will exclude all xml files
+  :param post_copy: callable that accepts source and target paths and will be called after copying
+  """
+  for item in os.listdir(src):
+    if exclude and item in exclude:
+      continue
+
+    _src = os.path.join(src, item)
+    _dest = os.path.join(dest, item)
+
+    if os.path.isdir(_src):
+      if not os.path.exists(_dest):
+        os.makedirs(_dest)
+      copy_tree(_src, _dest, exclude, post_copy)
+    else:
+      _dest_dirname = os.path.dirname(_dest)
+      if not os.path.exists(_dest_dirname):
+        os.makedirs(_dest_dirname)
+      shutil.copy(_src, _dest)
+
+    if post_copy:
+      post_copy(_src, _dest)
+
+
+def process_metainfo(file_path, config_data, stack_version_changes):
+  tree = ET.parse(file_path)
+  root = tree.getroot()
+
+  if root.find('versions') is not None or root.find('services') is None:
+    # process stack metainfo.xml
+    extends_tag = root.find('extends')
+    if extends_tag is not None:
+      version = extends_tag.text
+      if version in stack_version_changes:
+        extends_tag.text = stack_version_changes[version]
+        tree.write(file_path)
+
+    current_version = file_path.split(os.sep)[-2]
+    modify_active_tag = False
+    active_tag_value = None
+    for stack in config_data.versions:
+      if stack.version == current_version and 'active' in stack:
+        modify_active_tag = True
+        active_tag_value = stack.active
+        break
+
+    if modify_active_tag:
+      versions_tag = root.find('versions')
+      if versions_tag is None:
+        versions_tag = ET.SubElement(root, 'versions')
+      active_tag = versions_tag.find('active')
+      if active_tag is None:
+        active_tag = ET.SubElement(versions_tag, 'active')
+      active_tag.text = active_tag_value
+      tree.write(file_path)
+  else:
+    # process service metainfo.xml
+    services_tag = root.find('services')
+    if services_tag is not None:
+      for service_tag in services_tag.findall('service'):
+        name = service_tag.find('name').text
+        for stack in config_data.versions:
+          for service in stack.services:
+            if service.name == name:
+              if 'version' in service:
+                service_version_tag = service_tag.find('version')
+                if service_version_tag is None:
+                  service_version_tag = ET.SubElement(service_tag, 'version')
+                service_version_tag.text = service.version
+                tree.write(file_path)
+
+
+def process_upgrade_xml(file_path, target_version, config_data, stack_version_changes):
+  # change versions in xml
+  tree = ET.parse(file_path)
+  root = tree.getroot()
+
+  for target_tag in root.findall('target'):
+    version = '.'.join([el for el in target_tag.text.split('.') if el != '*'])
+    if version in stack_version_changes:
+      target_tag.text = target_tag.text.replace(version, stack_version_changes[version])
+      tree.write(file_path)
+
+  for target_tag in root.findall('target-stack'):
+    base_stack_name, base_stack_version = target_tag.text.split('-')
+    new_target_stack_text = target_tag.text.replace(base_stack_name, config_data.stackName)
+    if base_stack_version in stack_version_changes:
+      new_target_stack_text = new_target_stack_text.replace(base_stack_version,
+                                                            stack_version_changes[base_stack_version])
+    target_tag.text = new_target_stack_text
+    tree.write(file_path)
+
+  # rename upgrade files
+  if target_version in stack_version_changes:
+    new_name = os.path.join(os.path.dirname(file_path),
+                            'upgrade-{0}.xml'.format(stack_version_changes[target_version]))
+    os.rename(file_path, new_name)
+  pass
+
+
+def process_stack_advisor(file_path, config_data, stack_version_changes):
+  CLASS_NAME_REGEXP = '([A-Za-z]+)(\d+)StackAdvisor'
+
+  stack_advisor_content = open(file_path, 'r').read()
+
+  for stack_name, stack_version in  re.findall(CLASS_NAME_REGEXP, stack_advisor_content):
+    what = stack_name + stack_version + 'StackAdvisor'
+    stack_version_dotted = '.'.join(list(stack_version))
+    if stack_version_dotted in stack_version_changes:
+      to = config_data.stackName + stack_version_changes[stack_version_dotted].replace('.','') + 'StackAdvisor'
+    else:
+      to = config_data.stackName + stack_version + 'StackAdvisor'
+    stack_advisor_content = stack_advisor_content.replace(what, to)
+
+  with open(file_path, 'w') as f:
+    f.write(stack_advisor_content)
+
+def process_repoinfo_xml(file_path, config_data, stack_version_changes):
+  tree = ET.parse(file_path)
+  root = tree.getroot()
+  for baseurl_tag in root.iter('baseurl'):
+    baseurl_tag.text = 'http://examplerepo.com'
+
+  tree.write(file_path)
+
+def process_py_files(file_path, config_data, stack_version_changes):
+  file_content = open(file_path, 'r').read()
+  # replace select tools
+  file_content = file_content.replace('hdp-select', config_data.selectTool)
+  file_content = file_content.replace('conf-select', config_data.confSelectTool)
+
+  with open(file_path, 'w') as f:
+    f.write(file_content)
+
+def copy_stacks(resources_folder, output_folder, config_data):
+  original_folder = os.path.join(resources_folder, 'stacks', config_data.baseStackName)
+  target_folder = os.path.join(output_folder, 'stacks', config_data.stackName)
+  stack_version_changes = {}
+
+  for stack in config_data.versions:
+    if stack.version != stack.baseVersion:
+      stack_version_changes[stack.baseVersion] = stack.version
+
+  for stack in config_data.versions:
+    original_stack = os.path.join(original_folder, stack.baseVersion)
+    target_stack = os.path.join(target_folder, stack.version)
+
+    desired_services = [service.name for service in stack.services]
+    desired_services.append('stack_advisor.py')  # stack_advisor.py placed in stacks folder
+    base_stack_services = os.listdir(os.path.join(original_stack, 'services'))
+    ignored_services = [service for service in base_stack_services if service not in desired_services]
+
+    def post_copy(src, target):
+      # process metainfo.xml
+      if target.endswith('metainfo.xml'):
+        process_metainfo(target, config_data, stack_version_changes)
+        return
+      # process upgrade-x.x.xml
+      _upgrade_re = re.compile('upgrade-(.*)\.xml')
+      result = re.search(_upgrade_re, target)
+      if result:
+        target_version = result.group(1)
+        process_upgrade_xml(target, target_version, config_data, stack_version_changes)
+        return
+      # process stack_advisor.py
+      if target.endswith('stack_advisor.py'):
+        process_stack_advisor(target, config_data, stack_version_changes)
+        return
+      # process repoinfo.xml
+      if target.endswith('repoinfo.xml'):
+        process_repoinfo_xml(target, config_data, stack_version_changes)
+        return
+      # process python files
+      if target.endswith('.py'):
+        process_py_files(target, config_data, stack_version_changes)
+        return
+
+        # TODO add more processing here for *.py files and others
+
+    copy_tree(original_stack, target_stack, ignored_services, post_copy=post_copy)
+
+
+def main(argv):
+  HELP_STRING = 'WhiteLabelStackDefinition.py -c <config> -r <resources_folder> -o <output_folder>'
+  config = ''
+  resources_folder = ''
+  output_folder = ''
+  try:
+    opts, args = getopt.getopt(argv, "hc:o:r:", ["config=", "out=", "resources="])
+  except getopt.GetoptError:
+    print HELP_STRING
+    sys.exit(2)
+  for opt, arg in opts:
+    if opt == '-h':
+      print HELP_STRING
+      sys.exit()
+    elif opt in ("-c", "--config"):
+      config = arg
+    elif opt in ("-r", "--resources"):
+      resources_folder = arg
+    elif opt in ("-o", "--out"):
+      output_folder = arg
+  if not config or not resources_folder or not output_folder:
+    print HELP_STRING
+    sys.exit(2)
+  copy_stacks(resources_folder, output_folder, _named_dict(json.load(open(config, "r"))))
+
+
+if __name__ == "__main__":
+  main(sys.argv[1:])

ambari-common/src/main/python/white_labeling/__init__.py

@@ -0,0 +1,18 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""

ambari-common/src/main/python/white_labeling/test.json

@@ -0,0 +1,61 @@
+{
+  "stackName": "PHD",
+  "baseStackName": "HDP",
+  "selectTool": "phd-select",
+  "confSelectTool": "phd-conf-select",
+  "versions": [
+    {
+      "version": "2.0.6",
+      "baseVersion": "2.0.6",
+      "active": "false",
+      "services": [
+        {
+          "name": "HDFS"
+        },
+        {
+          "name": "ZOOKEEPER"
+        }
+      ]
+    },
+    {
+      "version": "2.1",
+      "baseVersion": "2.1",
+      "active": "false",
+      "services": [
+        {
+          "name": "HDFS"
+        },
+        {
+          "name": "ZOOKEEPER"
+        }
+      ]
+    },
+    {
+      "version": "3.0",
+      "baseVersion": "2.2",
+      "active": "true",
+      "services": [
+        {
+          "name": "HDFS",
+          "version": "2.6.0.2.2.0.0"
+        },
+        {
+          "name": "ZOOKEEPER"
+        }
+      ]
+    },
+    {
+      "version": "3.1",
+      "baseVersion": "2.3",
+      "active": "true",
+      "services": [
+        {
+          "name": "HDFS"
+        },
+        {
+          "name": "ZOOKEEPER"
+        }
+      ]
+    }
+  ]
+}