#!/usr/bin/env python
'''
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''

import optparse
import sys
import os
import logging
import tempfile
import json
import re
import base64
import time
import xml
import xml.etree.ElementTree as ET
import StringIO
import ConfigParser
from optparse import OptionGroup

logger = logging.getLogger('AmbariTakeoverConfigMerge')

CONFIG_MAPPING_HELP_TEXT = """
JSON file should content map with {regex_path : <service>-log4j}
Example:
{".+/hadoop/.+/log4j.properties" : "hdfs-log4j",
".+/etc/zookeeper/conf/log4j.properties" : "zookeeper-log4j"
"c6401.ambari.apache.org/etc/hive/conf/log4j.properties" : "hive-log4j"}
"""

LICENSE = """

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""


class Parser:
  pass

class ShParser(Parser):
  def read_data_to_map(self, path):
    with open(path, 'r') as file:
      file_content = file.read()
    return {"content" : file_content}, None

class YamlParser(Parser): # Used Yaml parser to read data into a map
  def read_data_to_map(self, path):
    try:
      import yaml
    except ImportError:
      logger.error("Module PyYAML not installed. Please try to execute \"pip install pyyaml\" for installing PyYAML module.")
      sys.exit(1)

    configurations = {}
    with open(path, 'r') as file:
      try:
        for name, value in yaml.load(file).iteritems():
          if name != None:
            configurations[name] = str(value)
      except:
        logger.error("Couldn't parse {0} file. Skipping ...".format(path))
        return None, None
    return configurations, None


class PropertiesParser(Parser): # Used ConfigParser parser to read data into a map
  def read_data_to_map(self, path):
    configurations = {}
    try :
      #Adding dummy section to properties file content for use ConfigParser
      properties_file_content = StringIO.StringIO()
      properties_file_content.write('[dummysection]\n')
      properties_file_content.write(open(path).read())
      properties_file_content.seek(0, os.SEEK_SET)

      cp = ConfigParser.ConfigParser()
      cp.readfp(properties_file_content)

      for section in cp._sections:
        for name, value in cp._sections[section].iteritems():
          if name != None:
            configurations[name] = value
        del configurations['__name__']
    except:
      logger.exception("ConfigParser error: ")
    return configurations, None


class XmlParser(Parser):  # Used DOM parser to read data into a map
  def read_data_to_map(self, path):
    configurations = {}
    properties_attributes = {}
    tree = ET.parse(path)
    root = tree.getroot()
    for properties in root.getiterator('property'):
      name = properties.find('name')
      value = properties.find('value')
      #TODO support all properties attributes
      final = properties.find('final')

      if name != None:
        name_text = name.text if name.text else ""
      else:
        logger.warn("No name is found for one of the properties in {0}, ignoring it".format(path))
        continue

      if value != None:
        value_text = value.text if value.text else ""
      else:
        logger.warn("No value is found for \"{0}\" in {1}, using empty string for it".format(name_text, path))
        value_text = ""

      if final != None:
        final_text = final.text if final.text else ""
        properties_attributes[name_text] = final_text

      configurations[name_text] = value_text
    logger.debug("Following configurations found in {0}:\n{1}".format(path, configurations))
    return configurations, properties_attributes

class ConfigMerge:

  CONTENT_UNKNOWN_FILES_MAPPING_FILE = {}
  LEFT_INPUT_DIR = "/tmp/left"
  RIGHT_INPUT_DIR = "/tmp/right"
  INPUT_DIR = '/etc/hadoop'
  OUTPUT_DIR = '/tmp'
  OUT_FILENAME = 'ambari_takeover_config_merge.out'
  JSON_FILENAME = 'ambari_takeover_config_merge.json'
  PARSER_BY_EXTENSIONS = {'.xml' : XmlParser(), '.yaml' : YamlParser(), '.properties' : PropertiesParser(), '.sh' : ShParser()}
  SUPPORTED_EXTENSIONS = ['.xml', '.yaml', '.properties', '.sh']
  SUPPORTED_FILENAME_ENDINGS = {".sh" : "-env"}
  UNKNOWN_FILES_MAPPING_FILE = None

  CONFIGS_WITH_CONTENT = ['pig-properties', '-log4j']

  NOT_MAPPED_FILES = ['log4j.properties']


  config_files_map = {}
  left_file_paths = None
  right_file_paths = None

  def __init__(self, config_files_map=None, left_file_paths=None, right_file_paths=None):
    self.config_files_map = config_files_map
    self.left_file_paths = left_file_paths
    self.right_file_paths = right_file_paths

  @staticmethod
  def get_all_supported_files_grouped_by_name(extensions=SUPPORTED_EXTENSIONS, directory=INPUT_DIR):
    filePaths = {}
    for dirName, subdirList, fileList in os.walk(directory, followlinks=True):
      for file in fileList:
        root, ext = os.path.splitext(file)
        if ext in extensions:
          file_path = os.path.join(dirName, file)
          if ext in ConfigMerge.SUPPORTED_FILENAME_ENDINGS and not ConfigMerge.SUPPORTED_FILENAME_ENDINGS[ext] in root:
            logger.warn("File {0} is not configurable by Ambari. Skipping...".format(file_path))
            continue
          config_name = None

          if ConfigMerge.UNKNOWN_FILES_MAPPING_FILE:
            for path_regex, name in ConfigMerge.CONTENT_UNKNOWN_FILES_MAPPING_FILE.iteritems():
              match = re.match(path_regex, os.path.relpath(file_path, ConfigMerge.INPUT_DIR))
              if match:
                config_name = name
                break

          if not config_name:
            if file in ConfigMerge.NOT_MAPPED_FILES:
              if ConfigMerge.UNKNOWN_FILES_MAPPING_FILE:
                logger.error("File {0} doesn't match any regex from {1}".format(file_path, ConfigMerge.UNKNOWN_FILES_MAPPING_FILE))
              else:
                logger.error("Cannot map {0} to Ambari config type. Please use -u option to specify config mapping for this file. \n"
                             "For more information use --help option for script".format(file_path))
              continue
            else:
              config_name = file

          if not config_name in filePaths:
            filePaths[config_name] = []
          filePaths[config_name].append((file_path, ConfigMerge.PARSER_BY_EXTENSIONS[ext]))
    return filePaths

  @staticmethod
  def merge_configurations(filepath_to_configurations):
    configuration_information_dict = {}
    property_name_to_value_to_filepaths = {}
    merged_configurations = {}

    for path, configurations in filepath_to_configurations.iteritems():
      for configuration_name, value in configurations.iteritems():
        if not configuration_name in property_name_to_value_to_filepaths:
          property_name_to_value_to_filepaths[configuration_name] = {}
        if not value in property_name_to_value_to_filepaths[configuration_name]:
          property_name_to_value_to_filepaths[configuration_name][value] = []

        logger.debug("Iterating over '{0}' with value '{1}' in file '{2}'".format(configuration_name, value, path))
        property_name_to_value_to_filepaths[configuration_name][value].append(path)
        merged_configurations[configuration_name] = value

    return merged_configurations, property_name_to_value_to_filepaths

  @staticmethod
  def format_for_blueprint(configurations, attributes):
    all_configs = []
    for configuration_type, configuration_properties in configurations.iteritems():
      is_content = False
      all_configs.append({})

      for config_with_content in ConfigMerge.CONFIGS_WITH_CONTENT:
        if config_with_content in configuration_type:
          is_content = True
          break

      if is_content:
        content = LICENSE
        for property_name, property_value in configuration_properties.iteritems():
          content+=property_name + "=" + property_value + "\n"
        all_configs[-1][configuration_type] = {'properties': {"content" : content}}
      else:
        all_configs[-1][configuration_type] = {'properties' :configuration_properties}

      for configuration_type_attributes, properties_attributes in attributes.iteritems():
        if properties_attributes and configuration_type == configuration_type_attributes:
          all_configs[-1][configuration_type].update({"properties_attributes" : {"final" : properties_attributes}})

    return {
      "configurations": all_configs,
      "host_groups": [],
      "Blueprints": {}
    }

  @staticmethod
  def format_conflicts_output(property_name_to_value_to_filepaths):
    output = ""
    for property_name, value_to_filepaths in property_name_to_value_to_filepaths.iteritems():
      if len(value_to_filepaths) == 1:
        continue

      first_item = False
      for value, filepaths in value_to_filepaths.iteritems():
        if not first_item:
          first_item = True
          output += "\n\n=== {0} | {1} | {2} |\nHas conflicts with:\n\n".format(property_name,filepaths[0], value)
          continue
        for filepath in filepaths:
          output += "| {0} | {1} | {2} |\n".format(property_name, filepath, value)

    return output

  def perform_merge(self):
    result_configurations = {}
    result_property_attributes = {}
    has_conflicts = False
    for filename, paths_and_parsers in self.config_files_map.iteritems():
      filepath_to_configurations = {}
      filepath_to_property_attributes = {}
      configuration_type = os.path.splitext(filename)[0]
      for path_and_parser in paths_and_parsers:
        path, parser = path_and_parser
        logger.debug("Read data from {0}".format(path))
        parsed_configurations_from_path, parsed_properties_attributes = parser.read_data_to_map(path)
        if parsed_configurations_from_path != None:
          filepath_to_configurations[path] = parsed_configurations_from_path
        if parsed_properties_attributes != None:
          filepath_to_property_attributes[path] = parsed_properties_attributes

      #configs merge
      merged_configurations, property_name_to_value_to_filepaths = ConfigMerge.merge_configurations(
        filepath_to_configurations)

      #properties attributes merge
      merged_attributes, property_name_to_attribute_to_filepaths = ConfigMerge.merge_configurations(
        filepath_to_property_attributes)

      configuration_conflicts_output = ConfigMerge.format_conflicts_output(property_name_to_value_to_filepaths)
      attribute_conflicts_output = ConfigMerge.format_conflicts_output(property_name_to_attribute_to_filepaths)

      if configuration_conflicts_output:
        has_conflicts = True
        conflict_filename = os.path.join(self.OUTPUT_DIR, configuration_type + "-conflicts.txt")
        logger.warn(
          "You have configurations conflicts for {0}. Please check {1}".format(configuration_type, conflict_filename))
        with open(conflict_filename, "w") as fp:
          fp.write(configuration_conflicts_output)

      if attribute_conflicts_output:
        has_conflicts = True
        conflict_filename = os.path.join(self.OUTPUT_DIR, configuration_type + "-attributes-conflicts.txt")
        logger.warn(
          "You have property attribute conflicts for {0}. Please check {1}".format(configuration_type, conflict_filename))
        with open(conflict_filename, "w") as fp:
          fp.write(attribute_conflicts_output)

      result_configurations[configuration_type] = merged_configurations
      result_property_attributes[configuration_type] = merged_attributes


    result_json_file = os.path.join(self.OUTPUT_DIR, "blueprint.json")
    logger.info("Using '{0}' file as output for blueprint template".format(result_json_file))

    with open(result_json_file, 'w') as outfile:
      outfile.write(json.dumps(ConfigMerge.format_for_blueprint(result_configurations, result_property_attributes), sort_keys=True, indent=4,
                               separators=(',', ': ')))
    if has_conflicts:
      logger.info("Script finished with configurations conflicts, please resolve them before using the blueprint")
      return 1
    else:
      logger.info("Script successfully finished")
      return 0

  def perform_diff(self):
    configurations_conflicts = {}
    attributes_conflicts = {}
    file_conflicts = []
    matches_configs = []

    for right_configs_names in self.right_file_paths:
      for left_configs_names in self.left_file_paths:
        if right_configs_names == left_configs_names:
          matches_configs.append(right_configs_names)

    for match_config in matches_configs:
      configurations_conflicts[match_config], attributes_conflicts[match_config] = ConfigMerge.configuration_diff(self.left_file_paths[match_config], self.right_file_paths[match_config])

    file_conflicts = ConfigMerge.get_missing_files(self.right_file_paths, matches_configs, ConfigMerge.LEFT_INPUT_DIR) + \
                     ConfigMerge.get_missing_files(self.left_file_paths, matches_configs, ConfigMerge.RIGHT_INPUT_DIR)

    configuration_diff_output = None
    configuration_diff_output = ConfigMerge.format_diff_output(file_conflicts, configurations_conflicts, attributes_conflicts)

    if configuration_diff_output and configuration_diff_output != "":
      conflict_filename = os.path.join(ConfigMerge.OUTPUT_DIR, "file-diff.txt")
      logger.warn(
        "You have file diff conflicts. Please check {0}".format(conflict_filename))
      with open(conflict_filename, "w") as fp:
        fp.write(configuration_diff_output)

    logger.info("Script successfully finished")
    return 0

  @staticmethod
  def format_diff_output(file_conflicts, configurations_conflicts, attributes_conflicts):
    output = ""
    if file_conflicts:
      output += "======= File diff conflicts ====== \n\n"
      for file_conflict in file_conflicts:
        output+=str(file_conflict)+"\n"

    if configurations_conflicts:
      output += "\n\n======= Property diff conflicts ====== "
      for config_name, property in configurations_conflicts.iteritems():
          if property:
            output+= "\n\n||| " + config_name + " |||\n"
            output+= "\n".join(str(p) for p in property)

    if attributes_conflicts:
      output += "\n\n======= Final attribute diff conflicts ====== "
      for config_name, property_with_attribute in attributes_conflicts.iteritems():
        if property_with_attribute:
          output+= "\n\n||| " + config_name + " |||\n"
          output+= "\n".join(str(p) for p in property_with_attribute)

    return output

  @staticmethod
  def configuration_diff(left, right):
    properties_conflicts = []
    attributes_conflicts = []
    left_path, left_parser = left[0]
    left_configurations, left_attributes = left_parser.read_data_to_map(left_path)
    right_path, right_parser = right[0]
    right_configurations, right_attributes = right_parser.read_data_to_map(right_path)

    matches_configs = []
    matches_attributes = []

    matches_configs, properties_conflicts = ConfigMerge.get_conflicts_and_matches(left_configurations, right_configurations, left_path, right_path)
    properties_conflicts += ConfigMerge.get_missing_properties(left_configurations, matches_configs, right_path) + \
                            ConfigMerge.get_missing_properties(right_configurations, matches_configs, left_path)

    if left_attributes and right_attributes:
      matches_attributes, attributes_conflicts = ConfigMerge.get_conflicts_and_matches(left_attributes, right_attributes, left_path, right_path)
      attributes_conflicts += ConfigMerge.get_missing_attributes(left_attributes, matches_attributes, right_path) + \
                              ConfigMerge.get_missing_attributes(right_attributes, matches_attributes, left_path)
    elif left_attributes:
      attributes_conflicts = ConfigMerge.get_missing_attributes(left_attributes, matches_attributes, right_path)

    elif right_attributes:
      attributes_conflicts = ConfigMerge.get_missing_attributes(right_attributes, matches_attributes, left_path)

    return properties_conflicts, attributes_conflicts

  @staticmethod
  def get_conflicts_and_matches(left_items, right_items, left_path, right_path):
    matches = []
    conflicts = []
    for left_key, left_value in left_items.iteritems():
      for right_key, right_value in right_items.iteritems():
        if left_key == right_key:
          matches.append(right_key)
          if left_value != right_value:
            conflicts.append({right_key : [{left_path : left_value}, {right_path :right_value}]})
    return matches, conflicts

  @staticmethod
  def get_missing_attributes(attributes, matches, file_path):
    conflicts = []
    for key, value in attributes.iteritems():
      if not key in matches:
        conflicts.append({key : "Final attribute is missing in {0} file".format(file_path)})
    return conflicts

  @staticmethod
  def get_missing_properties(configurations, matches, file_path):
    conflicts = []
    for key, value in configurations.iteritems():
      if not key in matches:
        conflicts.append({key : "Property is missing in {0} file".format(file_path)})
    return conflicts

  @staticmethod
  def get_missing_files(config_file_paths, matches, input_dir):
    conflicts = []
    for file_name in config_file_paths:
      if file_name not in matches:
        conflicts.append({file_name : "Configurations file is missing for {0} directory".format(input_dir)})
    return conflicts

def main():
  tempDir = tempfile.gettempdir()
  outputDir = os.path.join(tempDir)

  parser = optparse.OptionParser(usage="usage: %prog [options]")
  parser.set_description('This python program is an Ambari thin client and '
                         'supports Ambari cluster takeover by generating a '
                         'configuration json that can be used with a '
                         'blueprint.\n\nIt reads actual hadoop configs '
                         'from a target directory and produces an out file '
                         'with problems found that need to be addressed and '
                         'the json file which can be used to create the '
                         'blueprint.\n\nThis script only works with *.xml *.yaml '
                         'and *.properties extensions of files.')

  parser.add_option("-a", "--action", dest="action", default = "merge",
                    help="Script action. (merge/diff) [default: merge]")

  parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
                    default=False, help="output verbosity.")
  parser.add_option("-o", "--outputdir", dest="outputDir", default=outputDir,
                    metavar="FILE", help="Output directory. [default: /tmp]")
  parser.add_option("-u", '--unknown-files-mapping-file',dest="unknown_files_mapping_file",
                    metavar="FILE", help=CONFIG_MAPPING_HELP_TEXT, default="takeover_files_mapping.json")


  merge_options_group = OptionGroup(parser, "Required options for action 'merge'")
  merge_options_group.add_option("-i", "--inputdir", dest="inputDir", help="Input directory.")

  parser.add_option_group(merge_options_group)

  diff_options_group = OptionGroup(parser, "Required options for action 'diff'")
  diff_options_group.add_option("-l", "--leftInputDir", dest="leftInputDir", help="Left input directory.")
  diff_options_group.add_option("-r", "--rightInputDir", dest="rightInputDir", help="Right input directory.")

  parser.add_option_group(diff_options_group)

  (options, args) = parser.parse_args()

  # set verbose
  if options.verbose:
    logger.setLevel(logging.DEBUG)
  else:
    logger.setLevel(logging.INFO)

  ConfigMerge.OUTPUT_DIR = options.outputDir

  if not os.path.exists(ConfigMerge.OUTPUT_DIR):
    os.makedirs(ConfigMerge.OUTPUT_DIR)

  logegr_file_name = os.path.join(ConfigMerge.OUTPUT_DIR, "takeover_config_merge.log")

  file_handler = logging.FileHandler(logegr_file_name, mode="w")
  formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
  file_handler.setFormatter(formatter)
  logger.addHandler(file_handler)
  stdout_handler = logging.StreamHandler(sys.stdout)
  stdout_handler.setLevel(logging.INFO)
  stdout_handler.setFormatter(formatter)
  logger.addHandler(stdout_handler)

  #unknown file mapping
  if options.unknown_files_mapping_file and os.path.exists(options.unknown_files_mapping_file):
    ConfigMerge.UNKNOWN_FILES_MAPPING_FILE = options.unknown_files_mapping_file
    with open(options.unknown_files_mapping_file) as f:
      ConfigMerge.CONTENT_UNKNOWN_FILES_MAPPING_FILE = json.load(f)
  else:
    logger.warning("Config mapping file was not found at {0}. "
                   "Please provide it at the given path or provide a different path to it using -u option.".format(options.unknown_files_mapping_file))
  if options.action == "merge" :
    ConfigMerge.INPUT_DIR = options.inputDir
    file_paths = ConfigMerge.get_all_supported_files_grouped_by_name(ConfigMerge.INPUT_DIR)
    logger.info("Writing logs into '{0}' file".format(logegr_file_name))
    logger.debug("Following configuration files found:\n{0}".format(file_paths.items()))
    config_merge = ConfigMerge(config_files_map=file_paths)
    return config_merge.perform_merge()

  elif options.action == "diff" :
    if options.leftInputDir and os.path.isdir(options.leftInputDir):
      ConfigMerge.LEFT_INPUT_DIR = options.leftInputDir
    else:
      logger.error("Directory \"{0}\" doesn't exist. Use option \"-h\" for details".format(options.leftInputDir))
      return -1

    if options.rightInputDir and os.path.isdir(options.rightInputDir):
      ConfigMerge.RIGHT_INPUT_DIR = options.rightInputDir
    else:
      logger.error("Directory \"{0}\" doesn't exist. Use option \"-h\" for details".format(options.rightInputDir))
      return -1

    logger.info("Writing logs into '{0}' file".format(logegr_file_name))

    left_file_paths = ConfigMerge.get_all_supported_files_grouped_by_name(directory=ConfigMerge.LEFT_INPUT_DIR)
    logger.debug("Following configuration files found:\n{0} for left directory".format(left_file_paths.items()))
    right_file_paths = ConfigMerge.get_all_supported_files_grouped_by_name(directory=ConfigMerge.RIGHT_INPUT_DIR)
    logger.debug("Following configuration files found:\n{0} for right directory".format(right_file_paths.items()))
    config_merge = ConfigMerge(left_file_paths=left_file_paths , right_file_paths=right_file_paths)
    return config_merge.perform_diff()

  else:
    logger.error("Action \"{0}\" doesn't supports by script. Use option \"-h\" for details".format(options.action))
    return -1

if __name__ == "__main__":
  try:
    sys.exit(main())
  except (KeyboardInterrupt, EOFError):
    print("\nAborting ... Keyboard Interrupt.")
    sys.exit(1)
