Revert "AMBARI-18846 - Custom services should be able to easily specify their own dashboards"

This reverts commit 538af12f60e904be83d5504688c8e5136bb6765a.

ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py

@@ -153,8 +153,7 @@ class CustomServiceOrchestrator():
         self.file_cache.get_host_scripts_base_dir(server_url_prefix)          
         hook_dir = self.file_cache.get_hook_base_dir(command, server_url_prefix)
         base_dir = self.file_cache.get_service_base_dir(command, server_url_prefix)
-        self.file_cache.get_dashboard_base_dir(server_url_prefix)
-
+        
         script_path = self.resolve_script_path(base_dir, script)
         script_tuple = (script_path, base_dir)
 

ambari-agent/src/main/python/ambari_agent/FileCache.py

@@ -45,7 +45,6 @@ class FileCache():
   STACKS_CACHE_DIRECTORY="stacks"
   COMMON_SERVICES_DIRECTORY="common-services"
   CUSTOM_ACTIONS_CACHE_DIRECTORY="custom_actions"
-  DASHBOARD_DIRECTORY="dashboards"
   HOST_SCRIPTS_CACHE_DIRECTORY="host_scripts"
   HASH_SUM_FILE=".hash"
   ARCHIVE_NAME="archive.zip"
@@ -100,15 +99,6 @@ class FileCache():
                                   server_url_prefix)
 
 
-  def get_dashboard_base_dir(self, server_url_prefix):
-    """
-    Returns a base directory for dashboards
-    """
-    return self.provide_directory(self.cache_dir,
-                                  self.DASHBOARD_DIRECTORY,
-                                  server_url_prefix)
-
-
   def get_host_scripts_base_dir(self, server_url_prefix):
     """
     Returns a base directory for host scripts (host alerts, etc) which

ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py

@@ -220,7 +220,7 @@ class TestCustomServiceOrchestrator(TestCase):
     except AgentException:
       pass # Expected
 
-  @patch.object(FileCache, "get_dashboard_base_dir")
+
   @patch.object(CustomServiceOrchestrator, "resolve_script_path")
   @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
   @patch.object(FileCache, "get_host_scripts_base_dir")
@@ -234,8 +234,7 @@ class TestCustomServiceOrchestrator(TestCase):
                       get_hook_base_dir_mock, get_service_base_dir_mock, 
                       get_host_scripts_base_dir_mock, 
                       resolve_hook_script_path_mock, 
-                      resolve_script_path_mock,
-                      get_dashboard_base_dir_mock):
+                      resolve_script_path_mock):
     
     FileCache_mock.return_value = None
     command = {
@@ -266,7 +265,6 @@ class TestCustomServiceOrchestrator(TestCase):
     unix_process_id = 111
     orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
     get_hook_base_dir_mock.return_value = "/hooks/"
-    get_dashboard_base_dir_mock.return_value = "/dashboards/"
     # normal run case
     run_file_mock.return_value = {
         'stdout' : 'sss',
@@ -311,7 +309,6 @@ class TestCustomServiceOrchestrator(TestCase):
 
     pass
 
-  @patch.object(FileCache, "get_dashboard_base_dir")
   @patch("ambari_commons.shell.kill_process_with_children")
   @patch.object(CustomServiceOrchestrator, "resolve_script_path")
   @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
@@ -326,8 +323,7 @@ class TestCustomServiceOrchestrator(TestCase):
                       get_hook_base_dir_mock, get_service_base_dir_mock,
                       get_host_scripts_base_dir_mock,
                       resolve_hook_script_path_mock, resolve_script_path_mock,
-                      kill_process_with_children_mock,
-                      get_dashboard_base_dir_mock):
+                      kill_process_with_children_mock):
     FileCache_mock.return_value = None
     command = {
       'role' : 'REGION_SERVER',
@@ -357,7 +353,6 @@ class TestCustomServiceOrchestrator(TestCase):
     unix_process_id = 111
     orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
     get_hook_base_dir_mock.return_value = "/hooks/"
-    get_dashboard_base_dir_mock.return_value = "/dashboards/"
     run_file_mock_return_value = {
       'stdout' : 'killed',
       'stderr' : 'killed',

ambari-agent/src/test/python/ambari_agent/TestFileCache.py

@@ -117,18 +117,6 @@ class TestFileCache(TestCase):
       "('/var/lib/ambari-agent/cache', 'custom_actions', 'server_url_pref')")
     self.assertEquals(res, "dummy value")
 
-
-  @patch.object(FileCache, "provide_directory")
-  def test_get_dashboard_base_dir(self, provide_directory_mock):
-    provide_directory_mock.return_value = "dummy value"
-    fileCache = FileCache(self.config)
-    res = fileCache.get_dashboard_base_dir("server_url_pref")
-    self.assertEquals(
-      pprint.pformat(provide_directory_mock.call_args_list[0][0]),
-      "('/var/lib/ambari-agent/cache', 'dashboards', 'server_url_pref')")
-    self.assertEquals(res, "dummy value")
-
-
   @patch.object(FileCache, "build_download_url")
   def test_provide_directory_no_update(self, build_download_url_mock):
     try:

ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py

@@ -39,7 +39,6 @@ class ResourceFilesKeeper():
   COMMON_SERVICES_DIR="common-services"
   CUSTOM_ACTIONS_DIR="custom_actions"
   HOST_SCRIPTS_DIR="host_scripts"
-  DASHBOARDS_DIR="dashboards"
 
   # For these directories archives are created
   ARCHIVABLE_DIRS = [HOOKS_DIR, PACKAGE_DIR]
@@ -69,7 +68,7 @@ class ResourceFilesKeeper():
     """
     Performs housekeeping operations on resource files
     """
-    self.update_directory_archives()
+    self.update_directory_archieves()
     # probably, later we will need some additional operations
 
 
@@ -88,7 +87,7 @@ class ResourceFilesKeeper():
     # update the directories so that the .hash is generated
     self.update_directory_archive(archive_root)
 
-  def update_directory_archives(self):
+  def update_directory_archieves(self):
     """
     Please see AMBARI-4481 for more details
     """
@@ -113,9 +112,6 @@ class ResourceFilesKeeper():
     # agent host scripts
     self._update_resources_subdir_archive(self.HOST_SCRIPTS_DIR)
 
-    # custom service dashboards
-    self._update_resources_subdir_archive(self.DASHBOARDS_DIR)
-
 
   def _list_metainfo_dirs(self, root_dir):
     valid_items = []  # Format: <stack_dir, ignore(True|False)>
@@ -157,7 +153,7 @@ class ResourceFilesKeeper():
       if not self.nozip:
         self.zip_directory(directory, skip_empty_directory)
       # Skip generation of .hash file is directory is empty
-      if (skip_empty_directory and (not os.path.exists(directory) or not os.listdir(directory))):
+      if (skip_empty_directory and not os.listdir(directory)):
         self.dbg_out("Empty directory. Skipping generation of hash file for {0}".format(directory))
       else:
         self.write_hash_sum(directory, cur_hash)

ambari-server/src/main/python/ambari_server/serverConfiguration.py

@@ -397,7 +397,6 @@ class ServerConfigDefaults(object):
     self.EXTENSION_LOCATION_DEFAULT = ""
     self.COMMON_SERVICES_LOCATION_DEFAULT = ""
     self.MPACKS_STAGING_LOCATION_DEFAULT = ""
-    self.DASHBOARD_LOCATION_DEFAULT = ""
     self.SERVER_TMP_DIR_DEFAULT = ""
 
     self.DEFAULT_VIEWS_DIR = ""
@@ -469,7 +468,6 @@ class ServerConfigDefaultsWindows(ServerConfigDefaults):
     self.EXTENSION_LOCATION_DEFAULT = "resources\\extensions"
     self.COMMON_SERVICES_LOCATION_DEFAULT = "resources\\common-services"
     self.MPACKS_STAGING_LOCATION_DEFAULT = "resources\\mpacks"
-    self.DASHBOARD_LOCATION_DEFAULT = "resources\\dashboards"
     self.SERVER_TMP_DIR_DEFAULT = "data\\tmp"
 
     self.DEFAULT_VIEWS_DIR = "resources\\views"
@@ -556,7 +554,6 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults):
     self.EXTENSION_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/extensions")
     self.COMMON_SERVICES_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/common-services")
     self.MPACKS_STAGING_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/mpacks")
-    self.DASHBOARD_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/dashboards")
     self.SERVER_TMP_DIR_DEFAULT = AmbariPath.get("/var/lib/ambari-server/data/tmp")
 
     self.DEFAULT_VIEWS_DIR = AmbariPath.get("/var/lib/ambari-server/resources/views")
@@ -1432,14 +1429,6 @@ def get_mpacks_staging_location(properties):
     mpacks_staging_location = configDefaults.MPACKS_STAGING_LOCATION_DEFAULT
   return mpacks_staging_location
 
-
-#
-# Dashboard location
-#
-def get_dashboard_location(properties):
-  dashboard_location = configDefaults.DASHBOARD_LOCATION_DEFAULT
-  return dashboard_location
-
 #
 # Server temp location
 #

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py

@@ -31,7 +31,7 @@ import network
 
 GRAFANA_CONNECT_TRIES = 5
 GRAFANA_CONNECT_TIMEOUT = 10
-GRAFANA_SEARCH_BUILTIN_DASHBOARDS = "/api/search?tag=builtin"
+GRAFANA_SEARCH_BULTIN_DASHBOARDS = "/api/search?tag=builtin"
 GRAFANA_DATASOURCE_URL = "/api/datasources"
 GRAFANA_DASHBOARDS_URL = "/api/dashboards/db"
 METRICS_GRAFANA_DATASOURCE_NAME = "AMBARI_METRICS"
@@ -279,14 +279,14 @@ def create_ams_dashboards():
   Dashboard = namedtuple('Dashboard', ['uri', 'id', 'title', 'tags'])
 
   existing_dashboards = []
-  response = perform_grafana_get_call(GRAFANA_SEARCH_BUILTIN_DASHBOARDS, server)
+  response = perform_grafana_get_call(GRAFANA_SEARCH_BULTIN_DASHBOARDS, server)
   if response and response.status == 200:
     data = response.read()
     try:
       dashboards = json.loads(data)
     except:
       Logger.error("Unable to parse JSON response from grafana request: %s" %
-                   GRAFANA_SEARCH_BUILTIN_DASHBOARDS)
+                   GRAFANA_SEARCH_BULTIN_DASHBOARDS)
       Logger.info(data)
       return
 
@@ -302,7 +302,7 @@ def create_ams_dashboards():
   else:
     Logger.error("Failed to execute search query on Grafana dashboards. "
                  "query = %s\n statuscode = %s\n reason = %s\n data = %s\n" %
-                 (GRAFANA_SEARCH_BUILTIN_DASHBOARDS, response.status, response.reason, response.read()))
+                 (GRAFANA_SEARCH_BULTIN_DASHBOARDS, response.status, response.reason, response.read()))
     return
 
   Logger.debug('Dashboard definitions found = %s' % str(dashboard_files))

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py

@@ -75,9 +75,6 @@ dashboards_dirs.append(os.path.join(agent_cache_dir, service_package_folder,
 dashboards_dirs.append(os.path.join(agent_cache_dir, service_package_folder,
                                    'files', 'grafana-dashboards', 'default'))
 
-# Custom services
-dashboards_dirs.append(os.path.join(agent_cache_dir, 'dashboards', 'grafana-dashboards'))
-
 def get_grafana_dashboard_defs():
   dashboard_defs = []
   for dashboards_dir in dashboards_dirs:

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py

@@ -67,12 +67,11 @@ def format_Xmx_size_to_bytes(value, default='b'):
 # pre-splits based on selected services also passed as a parameter to the class.
 class FindSplitPointsForAMSRegions():
 
-  def __init__(self, ams_hbase_site, ams_hbase_env, serviceMetricsDir, customServiceMetricsDir,
+  def __init__(self, ams_hbase_site, ams_hbase_env, serviceMetricsDir,
                operation_mode = 'embedded', services = None):
     self.ams_hbase_site = ams_hbase_site
     self.ams_hbase_env = ams_hbase_env
     self.serviceMetricsDir = serviceMetricsDir
-    self.customServiceMetricsDir = customServiceMetricsDir
     self.services = services
     self.mode = operation_mode
     # Add host metrics if not present as input
@@ -118,19 +117,13 @@ class FindSplitPointsForAMSRegions():
     pass
 
   def initialize_ordered_set_of_metrics(self):
-    stackServiceFiles = [ f for f in os.listdir(self.serviceMetricsDir) if
+    onlyServicefiles = [ f for f in os.listdir(self.serviceMetricsDir) if
                   os.path.isfile(os.path.join(self.serviceMetricsDir, f)) ]
 
-    if os.path.exists(self.customServiceMetricsDir):
-      customServiceFiles = [ f for f in os.listdir(self.customServiceMetricsDir) if
-                  os.path.isfile(os.path.join(self.customServiceMetricsDir, f)) ]
-    else:
-      customServiceFiles = []
-
     metrics = set()
 
-    for file in stackServiceFiles:
-      # Process for stack services selected at deploy time or all stack services if
+    for file in onlyServicefiles:
+      # Process for services selected at deploy time or all services if
       # services arg is not passed
       if self.services is None or file.rstrip(metric_filename_ext) in self.services:
         print 'Processing file: %s' % os.path.join(self.serviceMetricsDir, file)
@@ -144,21 +137,6 @@ class FindSplitPointsForAMSRegions():
       pass
     pass
 
-    for file in customServiceFiles:
-      # Process for custom services selected at deploy time or all custom services if
-      # services arg is not passed
-      if self.services is None or file.rstrip(metric_filename_ext) in self.services:
-        print 'Processing file: %s' % os.path.join(self.customServiceMetricsDir, file)
-        service_metrics = set()
-        with open(os.path.join(self.customServiceMetricsDir, file), 'r') as f:
-          for metric in f:
-            service_metrics.add(metric.strip())
-          pass
-        pass
-        metrics.update(self.find_equidistant_metrics(list(sorted(service_metrics))))
-      pass
-    pass
-
     self.metrics = sorted(metrics)
     print 'metrics length: %s' % len(self.metrics)
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -783,7 +783,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     scriptDir = os.path.dirname(os.path.abspath(__file__))
     metricsDir = os.path.join(scriptDir, '../../../../common-services/AMBARI_METRICS/0.1.0/package')
     serviceMetricsDir = os.path.join(metricsDir, 'files', 'service-metrics')
-    customServiceMetricsDir = os.path.join(scriptDir, '../../../../dashboards/service-metrics')
     sys.path.append(os.path.join(metricsDir, 'scripts'))
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
 
@@ -805,7 +804,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       ams_hbase_env = configurations["ams-hbase-env"]["properties"]
 
     split_point_finder = FindSplitPointsForAMSRegions(
-      ams_hbase_site, ams_hbase_env, serviceMetricsDir, customServiceMetricsDir, operatingMode, servicesList)
+      ams_hbase_site, ams_hbase_env, serviceMetricsDir, operatingMode, servicesList)
 
     result = split_point_finder.get_split_points()
     precision_splits = ' '

ambari-server/src/test/python/TestResourceFilesKeeper.py

@@ -82,8 +82,7 @@ class TestResourceFilesKeeper(TestCase):
       "call('../resources/TestAmbaryServer.samples/" \
       "dummy_common_services/HIVE/0.11.0.2.0.5.0/package'),\n " \
       "call('../resources/custom_actions'),\n " \
-      "call('../resources/host_scripts'),\n " \
-      "call('../resources/dashboards')]"
+      "call('../resources/host_scripts')]"
   else:
     UPDATE_DIRECTORY_ARCHIVE_CALL_LIST = \
       "[call('..\\\\resources\\\\TestAmbaryServer.samples\\\\dummy_stack\\\\HIVE\\\\package'),\n " \
@@ -92,18 +91,17 @@ class TestResourceFilesKeeper(TestCase):
       "call('..\\\\resources\\\\TestAmbaryServer.samples\\\\dummy_common_services\\\\HIVE\\\\0.11.0.2.0.5.0\\\\package'),\n " \
       "call('..\\\\resources\\\\TestAmbaryServer.samples\\\\dummy_common_services\\\\HIVE\\\\0.11.0.2.0.5.0\\\\package'),\n " \
       "call('..\\\\resources\\\\custom_actions'),\n " \
-      "call('..\\\\resources\\\\host_scripts'),\n " \
-      "call('..\\\\resources\\\\dashboards')]"
+      "call('..\\\\resources\\\\host_scripts')]"
 
   def setUp(self):
     logging.basicConfig(level=logging.ERROR)
 
 
-  @patch.object(ResourceFilesKeeper, "update_directory_archives")
-  def test_perform_housekeeping(self, update_directory_archives_mock):
+  @patch.object(ResourceFilesKeeper, "update_directory_archieves")
+  def test_perform_housekeeping(self, update_directory_archieves_mock):
     resource_files_keeper = ResourceFilesKeeper(os.sep + "dummy-resources", os.sep + "dummy-path")
     resource_files_keeper.perform_housekeeping()
-    update_directory_archives_mock.assertCalled()
+    update_directory_archieves_mock.assertCalled()
     pass
 
 
@@ -111,7 +109,7 @@ class TestResourceFilesKeeper(TestCase):
   @patch.object(ResourceFilesKeeper, "list_common_services")
   @patch.object(ResourceFilesKeeper, "list_stacks")
   @patch("os.path.abspath")
-  def test_update_directory_archives(self, abspath_mock,
+  def test_update_directory_archieves(self, abspath_mock,
                                       list_active_stacks_mock,
                                       list_common_services_mock,
                                       update_directory_archive_mock):
@@ -122,7 +120,7 @@ class TestResourceFilesKeeper(TestCase):
                                               self.DUMMY_UNCHANGEABLE_COMMON_SERVICES]
     abspath_mock.side_effect = lambda s : s
     resource_files_keeper = ResourceFilesKeeper(self.TEST_RESOURCES_DIR, self.TEST_STACKS_DIR)
-    resource_files_keeper.update_directory_archives()
+    resource_files_keeper.update_directory_archieves()
     self.assertEquals(pprint.pformat(
       update_directory_archive_mock.call_args_list),
       self.UPDATE_DIRECTORY_ARCHIVE_CALL_LIST)