AMBARI-19018 - Services should be able to specify their own resources subdirs for syncing to agents

ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py

@@ -154,12 +154,7 @@ class CustomServiceOrchestrator():
         self.file_cache.get_host_scripts_base_dir(server_url_prefix)          
         hook_dir = self.file_cache.get_hook_base_dir(command, server_url_prefix)
         base_dir = self.file_cache.get_service_base_dir(command, server_url_prefix)
-        from ActionQueue import ActionQueue  # To avoid cyclic dependency
-        if self.COMMAND_TYPE in command and command[self.COMMAND_TYPE] == ActionQueue.EXECUTION_COMMAND:
-          logger.info("Found it - " + str(command[self.COMMAND_TYPE]) + " yeah")
-          # limiting to only EXECUTION_COMMANDs for now
-          # TODO need a design for limiting to specific role/component such as METRICS_GRAFANA
-          self.file_cache.get_dashboard_base_dir(server_url_prefix)
+        self.file_cache.get_custom_resources_subdir(command, server_url_prefix)
 
         script_path = self.resolve_script_path(base_dir, script)
         script_tuple = (script_path, base_dir)

ambari-agent/src/main/python/ambari_agent/FileCache.py

@@ -45,7 +45,6 @@ class FileCache():
   STACKS_CACHE_DIRECTORY="stacks"
   COMMON_SERVICES_DIRECTORY="common-services"
   CUSTOM_ACTIONS_CACHE_DIRECTORY="custom_actions"
-  DASHBOARD_DIRECTORY="dashboards"
   HOST_SCRIPTS_CACHE_DIRECTORY="host_scripts"
   HASH_SUM_FILE=".hash"
   ARCHIVE_NAME="archive.zip"
@@ -100,12 +99,17 @@ class FileCache():
                                   server_url_prefix)
 
 
-  def get_dashboard_base_dir(self, server_url_prefix):
+  def get_custom_resources_subdir(self, command, server_url_prefix):
     """
-    Returns a base directory for dashboards
+    Returns a custom directory which must be a subdirectory of the resources dir
     """
+    try:
+      custom_dir = command['commandParams']['custom_folder']
+    except KeyError:
+      return None
+
     return self.provide_directory(self.cache_dir,
-                                  self.DASHBOARD_DIRECTORY,
+                                  custom_dir,
                                   server_url_prefix)
 
 

ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py

@@ -220,7 +220,7 @@ class TestCustomServiceOrchestrator(TestCase):
     except AgentException:
       pass # Expected
 
-  @patch.object(FileCache, "get_dashboard_base_dir")
+  @patch.object(FileCache, "get_custom_resources_subdir")
   @patch.object(CustomServiceOrchestrator, "resolve_script_path")
   @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
   @patch.object(FileCache, "get_host_scripts_base_dir")
@@ -235,7 +235,7 @@ class TestCustomServiceOrchestrator(TestCase):
                       get_host_scripts_base_dir_mock, 
                       resolve_hook_script_path_mock, 
                       resolve_script_path_mock,
-                      get_dashboard_base_dir_mock):
+                      get_custom_resources_subdir_mock):
     
     FileCache_mock.return_value = None
     command = {
@@ -267,7 +267,6 @@ class TestCustomServiceOrchestrator(TestCase):
     unix_process_id = 111
     orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
     get_hook_base_dir_mock.return_value = "/hooks/"
-    get_dashboard_base_dir_mock.return_value = "/dashboards/"
     # normal run case
     run_file_mock.return_value = {
         'stdout' : 'sss',
@@ -278,7 +277,6 @@ class TestCustomServiceOrchestrator(TestCase):
     self.assertEqual(ret['exitcode'], 0)
     self.assertTrue(run_file_mock.called)
     self.assertEqual(run_file_mock.call_count, 3)
-    self.assertTrue(get_dashboard_base_dir_mock.called)
 
     run_file_mock.reset_mock()
 
@@ -301,25 +299,6 @@ class TestCustomServiceOrchestrator(TestCase):
 
     run_file_mock.reset_mock()
 
-    # For role=METRICS_GRAFANA, dashboards should be sync'd
-    command['role'] = 'METRICS_GRAFANA'
-    get_dashboard_base_dir_mock.reset_mock()
-    get_dashboard_base_dir_mock.return_value = "/dashboards/"
-
-    run_file_mock.return_value = {
-        'stdout' : 'sss',
-        'stderr' : 'eee',
-        'exitcode': 0,
-      }
-    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
-    self.assertEqual(ret['exitcode'], 0)
-    self.assertTrue(run_file_mock.called)
-    self.assertEqual(run_file_mock.call_count, 3)
-    self.assertTrue(get_dashboard_base_dir_mock.called)
-
-    command['role'] = 'REGION_SERVER'
-    run_file_mock.reset_mock()
-
     # unknown script type case
     command['commandParams']['script_type'] = "SOME_TYPE"
     ret = orchestrator.runCommand(command, "out.txt", "err.txt")
@@ -332,7 +311,6 @@ class TestCustomServiceOrchestrator(TestCase):
 
     pass
 
-  @patch.object(FileCache, "get_dashboard_base_dir")
   @patch("ambari_commons.shell.kill_process_with_children")
   @patch.object(CustomServiceOrchestrator, "resolve_script_path")
   @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
@@ -347,8 +325,7 @@ class TestCustomServiceOrchestrator(TestCase):
                       get_hook_base_dir_mock, get_service_base_dir_mock,
                       get_host_scripts_base_dir_mock,
                       resolve_hook_script_path_mock, resolve_script_path_mock,
-                      kill_process_with_children_mock,
-                      get_dashboard_base_dir_mock):
+                      kill_process_with_children_mock):
     FileCache_mock.return_value = None
     command = {
       'role' : 'REGION_SERVER',
@@ -378,7 +355,6 @@ class TestCustomServiceOrchestrator(TestCase):
     unix_process_id = 111
     orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
     get_hook_base_dir_mock.return_value = "/hooks/"
-    get_dashboard_base_dir_mock.return_value = "/dashboards/"
     run_file_mock_return_value = {
       'stdout' : 'killed',
       'stderr' : 'killed',

ambari-agent/src/test/python/ambari_agent/TestFileCache.py

@@ -119,10 +119,16 @@ class TestFileCache(TestCase):
 
 
   @patch.object(FileCache, "provide_directory")
-  def test_get_dashboard_base_dir(self, provide_directory_mock):
+  def test_get_custom_resources_subdir(self, provide_directory_mock):
     provide_directory_mock.return_value = "dummy value"
     fileCache = FileCache(self.config)
-    res = fileCache.get_dashboard_base_dir("server_url_pref")
+    command = {
+      'commandParams': {
+        'custom_folder' : 'dashboards'
+      }
+    }
+
+    res = fileCache.get_custom_resources_subdir(command, "server_url_pref")
     self.assertEquals(
       pprint.pformat(provide_directory_mock.call_args_list[0][0]),
       "('/var/lib/ambari-agent/cache', 'dashboards', 'server_url_pref')")

ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java

@@ -343,6 +343,7 @@ public class ExecutionCommand extends AgentCommand {
     String SCRIPT_TYPE = "script_type";
     String SERVICE_PACKAGE_FOLDER = "service_package_folder";
     String HOOKS_FOLDER = "hooks_folder";
+    String CUSTOM_FOLDER = "custom_folder";
     String STACK_NAME = "stack_name";
     String SERVICE_TYPE = "service_type";
     String STACK_VERSION = "stack_version";

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java

@@ -25,6 +25,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_RETRY_ENABLED;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_FOLDER;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
@@ -2217,6 +2218,14 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
       serviceInfo.getServicePackageFolder());
     commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
 
+    String customCacheDirectory = componentInfo.getCustomFolder();
+    if (customCacheDirectory != null) {
+      File customCache = new File(configs.getResourceDirPath(), customCacheDirectory);
+      if (customCache.exists() && customCache.isDirectory()) {
+        commandParams.put(CUSTOM_FOLDER, customCacheDirectory);
+      }
+    }
+
     String clusterName = cluster.getClusterName();
     if (customCommandExecutionHelper.isTopologyRefreshRequired(roleCommand.name(), clusterName, serviceName)) {
       commandParams.put(ExecutionCommand.KeyNames.REFRESH_TOPOLOGY, "True");

ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java

@@ -133,6 +133,9 @@ public class ComponentInfo {
 
   private String timelineAppid;
 
+  @XmlElement(name="customFolder")
+  private String customFolder;
+
   public ComponentInfo() {
   }
 
@@ -158,6 +161,7 @@ public class ComponentInfo {
     clientConfigFiles = prototype.clientConfigFiles;
     timelineAppid = prototype.timelineAppid;
     reassignAllowed = prototype.reassignAllowed;
+    customFolder = prototype.customFolder;
   }
 
   public String getName() {
@@ -396,6 +400,14 @@ public class ComponentInfo {
     this.reassignAllowed = reassignAllowed;
   }
 
+  public String getCustomFolder() {
+    return customFolder;
+  }
+
+  public void setCustomFolder(String customFolder) {
+    this.customFolder = customFolder;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;
@@ -426,6 +438,7 @@ public class ComponentInfo {
     if (name != null ? !name.equals(that.name) : that.name != null) return false;
     if (clientConfigFiles != null ? !clientConfigFiles.equals(that.clientConfigFiles) :
         that.clientConfigFiles != null) return false;
+    if (customFolder != null ? !customFolder.equals(that.customFolder) : that.customFolder != null) return false;
 
     return true;
   }
@@ -450,6 +463,7 @@ public class ComponentInfo {
     result = 31 * result + (clientConfigFiles != null ? clientConfigFiles.hashCode() : 0);
     // NULL = 0, TRUE = 2, FALSE = 1
     result = 31 * result + (versionAdvertisedField != null ? (versionAdvertisedField.booleanValue() ? 2 : 1) : 0);
+    result = 31 * result + (customFolder != null ? customFolder.hashCode() : 0);
     return result;
   }
 

ambari-server/src/main/python/ambari_server/serverConfiguration.py

@@ -186,7 +186,6 @@ SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server s
 DEFAULT_DB_NAME = "ambari"
 
 SECURITY_KEYS_DIR = "security.server.keys_dir"
-DASHBOARD_PATH_PROPERTY = 'dashboards.path'
 EXTENSION_PATH_PROPERTY = 'extensions.path'
 COMMON_SERVICES_PATH_PROPERTY = 'common.services.path'
 MPACKS_STAGING_PATH_PROPERTY = 'mpacks.staging.path'
@@ -398,8 +397,8 @@ class ServerConfigDefaults(object):
     self.EXTENSION_LOCATION_DEFAULT = ""
     self.COMMON_SERVICES_LOCATION_DEFAULT = ""
     self.MPACKS_STAGING_LOCATION_DEFAULT = ""
-    self.DASHBOARD_LOCATION_DEFAULT = ""
     self.SERVER_TMP_DIR_DEFAULT = ""
+    self.DASHBOARD_DIRNAME = "dashboards"
 
     self.DEFAULT_VIEWS_DIR = ""
 
@@ -470,7 +469,6 @@ class ServerConfigDefaultsWindows(ServerConfigDefaults):
     self.EXTENSION_LOCATION_DEFAULT = "resources\\extensions"
     self.COMMON_SERVICES_LOCATION_DEFAULT = "resources\\common-services"
     self.MPACKS_STAGING_LOCATION_DEFAULT = "resources\\mpacks"
-    self.DASHBOARD_LOCATION_DEFAULT = "resources\\dashboards"
     self.SERVER_TMP_DIR_DEFAULT = "data\\tmp"
 
     self.DEFAULT_VIEWS_DIR = "resources\\views"
@@ -557,7 +555,6 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults):
     self.EXTENSION_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/extensions")
     self.COMMON_SERVICES_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/common-services")
     self.MPACKS_STAGING_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/mpacks")
-    self.DASHBOARD_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/dashboards")
     self.SERVER_TMP_DIR_DEFAULT = AmbariPath.get("/var/lib/ambari-server/data/tmp")
 
     self.DEFAULT_VIEWS_DIR = AmbariPath.get("/var/lib/ambari-server/resources/views")
@@ -1438,13 +1435,8 @@ def get_mpacks_staging_location(properties):
 # Dashboard location
 #
 def get_dashboard_location(properties):
-  try:
-    dashboard_location = properties[DASHBOARD_PATH_PROPERTY]
-  except KeyError:
-    dashboard_location = configDefaults.DASHBOARD_LOCATION_DEFAULT
-
-  if not dashboard_location:
-    dashboard_location = configDefaults.DASHBOARD_LOCATION_DEFAULT
+  resources_dir = get_resources_location(properties)
+  dashboard_location = os.path.join(resources_dir, configDefaults.DASHBOARD_DIRNAME)
   return dashboard_location
 
 #

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml

@@ -101,6 +101,7 @@
           <category>MASTER</category>
           <cardinality>0-1</cardinality>
           <versionAdvertised>false</versionAdvertised>
+          <customFolder>dashboards</customFolder>
           <commandScript>
             <script>scripts/metrics_grafana.py</script>
             <scriptType>PYTHON</scriptType>

ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java

@@ -1281,6 +1281,15 @@ public class AmbariManagementControllerTest {
     assertEquals("false", ec.getCommandParams().get("command_retry_enabled"));
     Map<String, Set<String>> chInfo = ec.getClusterHostInfo();
     assertTrue(chInfo.containsKey("namenode_host"));
+    assertFalse(ec.getCommandParams().containsKey("custom_folder"));
+
+    ec = controller.getExecutionCommand(cluster,
+                                        s1.getServiceComponent("DATANODE").getServiceComponentHost(host1),
+                                        RoleCommand.START);
+    assertEquals(cluster1, ec.getClusterName());
+    assertNotNull(ec.getCommandParams());
+    assertTrue(ec.getCommandParams().containsKey("custom_folder"));
+    assertEquals("dashboards", ec.getCommandParams().get("custom_folder"));
   }
 
   @Test

ambari-server/src/test/java/org/apache/ambari/server/orm/InMemoryDefaultTestModule.java

@@ -90,6 +90,7 @@ public class InMemoryDefaultTestModule extends AbstractModule {
     String stacks = "src/test/resources/stacks";
     String version = "src/test/resources/version";
     String sharedResourcesDir = "src/test/resources/";
+    String resourcesDir = "src/test/resources/";
     if (System.getProperty("os.name").contains("Windows")) {
       stacks = ClassLoader.getSystemClassLoader().getResource("stacks").getPath();
       version = new File(new File(ClassLoader.getSystemClassLoader().getResource("").getPath()).getParent(), "version").getPath();
@@ -116,6 +117,10 @@ public class InMemoryDefaultTestModule extends AbstractModule {
       properties.setProperty(Configuration.SHARED_RESOURCES_DIR.getKey(), sharedResourcesDir);
     }
 
+    if (!properties.containsKey(Configuration.RESOURCES_DIR.getKey())) {
+      properties.setProperty(Configuration.RESOURCES_DIR.getKey(), resourcesDir);
+    }
+
     try {
       install(Modules.override(new BeanDefinitionsCachingTestControllerModule(properties)).with(new AbstractModule() {
         @Override

ambari-server/src/test/python/TestMpacks.py

@@ -64,7 +64,7 @@ def get_configs():
     serverConfiguration.STACK_LOCATION_KEY : "/var/lib/ambari-server/resources/stacks",
     serverConfiguration.COMMON_SERVICES_PATH_PROPERTY : "/var/lib/ambari-server/resources/common-services",
     serverConfiguration.EXTENSION_PATH_PROPERTY : "/var/lib/ambari-server/resources/extensions",
-    serverConfiguration.DASHBOARD_PATH_PROPERTY : "/var/lib/ambari-server/resources/dashboards",
+    serverConfiguration.RESOURCES_DIR_PROPERTY : "/var/lib/ambari-server/resources",
     serverConfiguration.MPACKS_STAGING_PATH_PROPERTY : mpacks_directory,
     serverConfiguration.SERVER_TMP_DIR_PROPERTY : "/tmp",
     serverConfiguration.JDBC_DATABASE_PROPERTY: "postgres"
@@ -100,9 +100,11 @@ class TestMpacks(TestCase):
       fail = True
     self.assertTrue(fail)
 
+  @patch("os.path.exists")
   @patch("ambari_server.setupMpacks.get_YN_input")
   @patch("ambari_server.setupMpacks.run_mpack_install_checker")
-  def test_validate_purge(self, run_mpack_install_checker_mock, get_YN_input_mock):
+  @patch("ambari_server.setupMpacks.get_ambari_properties")
+  def test_validate_purge(self, get_ambari_properties_mock, run_mpack_install_checker_mock, get_YN_input_mock, os_path_exists_mock):
     options = self._create_empty_options_mock()
     options.purge = True
     purge_list = options.purge_list.split(',')
@@ -112,6 +114,7 @@ class TestMpacks(TestCase):
     replay_mode = False
     run_mpack_install_checker_mock.return_value = (0, "No errors found", "")
     get_YN_input_mock.return_value = True
+    os_path_exists_mock.return_value = True
 
     fail = False
     try:
@@ -160,22 +163,28 @@ class TestMpacks(TestCase):
     extensions_directory = configs[serverConfiguration.EXTENSION_PATH_PROPERTY]
     common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
     mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
-    os_path_exists_mock.return_value = False
+    os_path_exists_mock.side_effect = [True]
 
     purge_stacks_and_mpacks(None)
-    os_path_exists_calls = []
+    os_path_exists_calls = [
+      call('/var/lib/ambari-server/resources'),
+    ]
     os_path_exists_mock.assert_has_calls(os_path_exists_calls)
 
+    os_path_exists_mock.side_effect = [True, False, False]
     purge_stacks_and_mpacks(options.purge_list.split(","))
     os_path_exists_calls = [
+      call('/var/lib/ambari-server/resources'),
       call(stacks_directory),
       call(mpacks_directory)
     ]
     os_path_exists_mock.assert_has_calls(os_path_exists_calls)
 
     options.purge_list = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
+    os_path_exists_mock.side_effect = [True, False, False, False]
     purge_stacks_and_mpacks(options.purge_list.split(","))
     os_path_exists_calls = [
+      call('/var/lib/ambari-server/resources'),
       call(stacks_directory),
       call(common_services_directory),
       call(mpacks_directory)
@@ -183,8 +192,10 @@ class TestMpacks(TestCase):
     os_path_exists_mock.assert_has_calls(os_path_exists_calls)
 
     options.purge_list = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, EXTENSION_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
+    os_path_exists_mock.side_effect = [True, False, False, False]
     purge_stacks_and_mpacks(options.purge_list.split(","))
     os_path_exists_calls = [
+      call('/var/lib/ambari-server/resources'),
       call(stacks_directory),
       call(extensions_directory),
       call(mpacks_directory)
@@ -193,10 +204,13 @@ class TestMpacks(TestCase):
 
     options.purge_list = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
     options.replay_mode = True
+    os_path_exists_mock.side_effect = [True, False, False, False]
     purge_stacks_and_mpacks(options.purge_list.split(","))
     os_path_exists_calls = [
+      call('/var/lib/ambari-server/resources'),
       call(stacks_directory),
-      call(common_services_directory)
+      call(common_services_directory),
+      call(mpacks_directory)
     ]
     os_path_exists_mock.assert_has_calls(os_path_exists_calls)
 
@@ -266,10 +280,11 @@ class TestMpacks(TestCase):
     get_ambari_version_mock.return_value = "2.4.0.0"
     run_os_command_mock.return_value = (0, "", "")
     mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
-    """
+
     os_path_exists_calls = [call('/tmp/mystack.tar.gz'),
                             call('mpacks/mystack-ambari-mpack-1.0.0.0/mpack.json'),
                             call('mpacks/mystack-ambari-mpack-1.0.0.0/hooks/before_install.py'),
+                            call('/var/lib/ambari-server/resources'),
                             call('/var/lib/ambari-server/resources/stacks'),
                             call('/var/lib/ambari-server/resources/extensions'),
                             call('/var/lib/ambari-server/resources/common-services'),
@@ -277,12 +292,14 @@ class TestMpacks(TestCase):
                             call(mpacks_directory + '/cache'),
                             call('/var/lib/ambari-server/resources/dashboards'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0'),
+                            call('/var/lib/ambari-server/resources'),
                             call('/var/lib/ambari-server/resources/common-services/SERVICEA'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEA/1.0/dashboards'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEA/2.0/dashboards'),
                             call('/var/lib/ambari-server/resources/common-services/SERVICEB'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEB/1.0.0/dashboards'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEB/2.0.0/dashboards'),
+                            call('/var/lib/ambari-server/resources'),
                             call('/var/lib/ambari-server/resources/stacks/MYSTACK'),
                             call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
                             call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'),
@@ -295,22 +312,26 @@ class TestMpacks(TestCase):
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/stacks/MYSTACK/2.0/services/SERVICEA/dashboards'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/stacks/MYSTACK/2.0/services/SERVICEB/dashboards'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/hooks/after_install.py')]
-    """
-    os_path_exists_mock.side_effect = [True, True, True, False, True, False, False, False, False,
-                                       False, False, False, False, False, False,
-                                       False, False, False, False, False, False, False, False,
-                                       False, False, False, False, True]
+
+    os_path_exists_mock.side_effect = [True, True, True, True, False, True, False, False, False, False,
+                                       False, True, False, False, False, False, False, False, True, False,
+                                       False, False, False, False, False, False, False, False, False, False,
+                                       True]
     get_ambari_properties_mock.return_value = configs
     shutil_move_mock.return_value = True
 
-    install_mpack(options)
+    try:
+      install_mpack(options)
+    except Exception as e:
+      print e
 
     stacks_directory = configs[serverConfiguration.STACK_LOCATION_KEY]
     common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
     extensions_directory = configs[serverConfiguration.EXTENSION_PATH_PROPERTY]
     mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
-    dashboards_directory = serverConfiguration.get_dashboard_location(configs)
     mpacks_staging_directory = os.path.join(mpacks_directory, "mystack-ambari-mpack-1.0.0.0")
+    resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
+    dashboards_directory = os.path.join(resources_directory, "dashboards")
 
     run_os_command_calls = [
       call([
@@ -377,6 +398,7 @@ class TestMpacks(TestCase):
            "SERVICEB", None)
     ]
 
+    os_path_exists_mock.assert_has_calls(os_path_exists_calls)
     self.assertTrue(purge_stacks_and_mpacks_mock.called)
     run_os_command_mock.assert_has_calls(run_os_command_calls)
     os_mkdir_mock.assert_has_calls(os_mkdir_calls)
@@ -403,8 +425,8 @@ class TestMpacks(TestCase):
     expand_mpack_mock.return_value = "mpacks/myextension-ambari-mpack-1.0.0.0"
     get_ambari_version_mock.return_value = "2.4.0.0"
 
-    os_path_exists_mock.side_effect = [True, True, True, False, True, False, False, False,
-                                       False, True, False, False, False]
+    os_path_exists_mock.side_effect = [True, True, True, True, False, True, False, False, False,
+                                       False, True, True, False, False, False]
     get_ambari_properties_mock.return_value = configs
     shutil_move_mock.return_value = True
 
@@ -413,10 +435,12 @@ class TestMpacks(TestCase):
     extensions_directory = configs[serverConfiguration.EXTENSION_PATH_PROPERTY]
     mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
     mpacks_staging_directory = os.path.join(mpacks_directory, "myextension-ambari-mpack-1.0.0.0")
-    dashboards_directory = serverConfiguration.get_dashboard_location(configs)
+    resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
+    dashboards_directory = os.path.join(resources_directory, "dashboards")
 
     os_path_exists_calls = [call('/tmp/myextension.tar.gz'),
                             call('mpacks/myextension-ambari-mpack-1.0.0.0/mpack.json'),
+                            call('/var/lib/ambari-server/resources'),
                             call('/var/lib/ambari-server/resources/stacks'),
                             call('/var/lib/ambari-server/resources/extensions'),
                             call('/var/lib/ambari-server/resources/common-services'),
@@ -424,6 +448,7 @@ class TestMpacks(TestCase):
                             call(mpacks_directory + '/cache'),
                             call('/var/lib/ambari-server/resources/dashboards'),
                             call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0'),
+                            call('/var/lib/ambari-server/resources'),
                             call('/var/lib/ambari-server/resources/extensions'),
                             call('/var/lib/ambari-server/resources/extensions/MYEXTENSION'),
                             call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0/extensions/MYEXTENSION/1.0/services'),
@@ -476,9 +501,9 @@ class TestMpacks(TestCase):
     expand_mpack_mock.return_value = "mpacks/myservice-ambari-mpack-1.0.0.0"
     get_ambari_version_mock.return_value = "2.4.0.0"
 
-    os_path_exists_mock.side_effect = [True, True, True, True, True, True,
-                                       True, True, False, False, False, False,
-                                       True, True, True, False, True, True,
+    os_path_exists_mock.side_effect = [True, True, True, True, True, True, True,
+                                       True, True, False, False, True, False, False,
+                                       True, True, True, True, False, True, True,
                                        True, False]
 
     get_ambari_properties_mock.return_value = configs
@@ -491,11 +516,13 @@ class TestMpacks(TestCase):
     common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
     mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
     mpacks_staging_directory = os.path.join(mpacks_directory, "myservice-ambari-mpack-1.0.0.0")
-    dashboards_directory = serverConfiguration.get_dashboard_location(configs)
+    resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
+    dashboards_directory = os.path.join(resources_directory, "dashboards")
 
     os_path_exists_calls = [call('/tmp/myservice.tar.gz'),
                             call('mpacks/myservice-ambari-mpack-1.0.0.0/mpack.json'),
                             call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
+                            call('/var/lib/ambari-server/resources'),
                             call('/var/lib/ambari-server/resources/stacks'),
                             call('/var/lib/ambari-server/resources/extensions'),
                             call('/var/lib/ambari-server/resources/common-services'),
@@ -503,8 +530,10 @@ class TestMpacks(TestCase):
                             call(mpacks_directory + '/cache'),
                             call('/var/lib/ambari-server/resources/dashboards'),
                             call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0'),
+                            call('/var/lib/ambari-server/resources'),
                             call('/var/lib/ambari-server/resources/common-services/MYSERVICE'),
                             call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/common-services/MYSERVICE/1.0.0/dashboards'),
+                            call('/var/lib/ambari-server/resources'),
                             call('/var/lib/ambari-server/resources/stacks/MYSTACK'),
                             call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
                             call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'),
@@ -562,14 +591,14 @@ class TestMpacks(TestCase):
     get_ambari_version_mock.return_value = "2.4.0.0"
     run_os_command_mock.return_value = (0, "", "")
     mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
-    os_path_exists_mock.side_effect = [True, True, True, True, True, True, True, True,
-                                       True, True, True, True, True, True, True, False,
-                                       False, True, False, False, True, False, False,
-                                       False, False, False, True, True, True, False,
-                                       True, True, False, True, True, False, False,
-                                       False, False, False, True, True, True, True,
-                                       True, True, True, False, True, False, True, True,
-                                       True, True, True, True]
+    os_path_exists_mock.side_effect = [True, True, True, True, True, True, True, True, True, True,
+                                       True, True, True, True, True, True, True, False, False, True,
+                                       True, False, False, True, False, False, False, False, False, True,
+                                       True, True, True, False, True, True, False, True, True, False,
+                                       False, False, False, False, True, True, True, True, True, True,
+                                       True, False, True, False, True, True, True, True, True, True,
+                                       True]
+
     get_ambari_properties_mock.return_value = configs
     shutil_move_mock.return_value = True
 
@@ -579,10 +608,12 @@ class TestMpacks(TestCase):
     common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
     mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
     mpacks_staging_directory = os.path.join(mpacks_directory, "mystack-ambari-mpack-1.0.0.1")
-    dashboards_directory = serverConfiguration.get_dashboard_location(configs)
+    resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
+    dashboards_directory = os.path.join(resources_directory, "dashboards")
 
     os_path_exists_calls = [call('/tmp/mystack-1.0.0.1.tar.gz'),
                             call('mpacks/mystack-ambari-mpack-1.0.0.1/mpack.json'),
+                            call('/var/lib/ambari-server/resources'),
                             call(mpacks_directory),
                             call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0/mpack.json'),
                             call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/mpack.json'),
@@ -591,6 +622,7 @@ class TestMpacks(TestCase):
                             call('/tmp/mystack-1.0.0.1.tar.gz'),
                             call('mpacks/mystack-ambari-mpack-1.0.0.1/mpack.json'),
                             call('mpacks/mystack-ambari-mpack-1.0.0.1/hooks/before_upgrade.py'),
+                            call('/var/lib/ambari-server/resources'),
                             call('/var/lib/ambari-server/resources/stacks'),
                             call('/var/lib/ambari-server/resources/extensions'),
                             call('/var/lib/ambari-server/resources/common-services'),
@@ -598,6 +630,7 @@ class TestMpacks(TestCase):
                             call(mpacks_directory + '/cache'),
                             call('/var/lib/ambari-server/resources/dashboards'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1'),
+                            call('/var/lib/ambari-server/resources'),
                             call('/var/lib/ambari-server/resources/common-services/SERVICEA'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEA/1.0/dashboards'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEA/2.0/dashboards'),
@@ -607,6 +640,7 @@ class TestMpacks(TestCase):
                             call('/var/lib/ambari-server/resources/common-services/SERVICEC'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEC/1.0.0/dashboards'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEC/2.0.0/dashboards'),
+                            call('/var/lib/ambari-server/resources'),
                             call('/var/lib/ambari-server/resources/stacks/MYSTACK'),
                             call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
                             call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'),
@@ -631,6 +665,7 @@ class TestMpacks(TestCase):
                             call('/var/lib/ambari-server/resources/dashboards/grafana-dashboards/SERVICEC'),
                             call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/3.0/services/SERVICEC/dashboards/service-metrics/SERVICEC.txt'),
                             call('/var/lib/ambari-server/resources/dashboards/service-metrics/SERVICEC.txt'),
+                            call('/var/lib/ambari-server/resources'),
                             call(mpacks_directory),
                             call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0/mpack.json'),
                             call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/mpack.json'),
@@ -775,7 +810,7 @@ class TestMpacks(TestCase):
       serverConfiguration.COMMON_SERVICES_PATH_PROPERTY : os.path.join(uninstall_directory, "common-services"),
       serverConfiguration.EXTENSION_PATH_PROPERTY : os.path.join(uninstall_directory, "extensions"),
       serverConfiguration.MPACKS_STAGING_PATH_PROPERTY : mpacks_directory,
-      serverConfiguration.DASHBOARD_PATH_PROPERTY : os.path.join(uninstall_directory, "dashboards"),
+      serverConfiguration.RESOURCES_DIR_PROPERTY : uninstall_directory,
       serverConfiguration.SERVER_TMP_DIR_PROPERTY : "/tmp"
     }
 
@@ -784,7 +819,8 @@ class TestMpacks(TestCase):
     stacks_directory = fake_configs[serverConfiguration.STACK_LOCATION_KEY]
     extension_directory = fake_configs[serverConfiguration.EXTENSION_PATH_PROPERTY]
     common_services_directory = fake_configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
-    dashboard_directory = fake_configs[serverConfiguration.DASHBOARD_PATH_PROPERTY]
+    resources_directory = fake_configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
+    dashboards_directory = os.path.join(resources_directory, "dashboards")
 
     _uninstall_mpack("mystack-ambari-mpack", "1.0.0.1")
 
@@ -794,8 +830,8 @@ class TestMpacks(TestCase):
                               call(os.path.join(stacks_directory, "2.0/files/metainfo2.xml")),
                               call(os.path.join(extension_directory, "SERVICEB")),
                               call(os.path.join(common_services_directory, "SERVICEB")),
-                              call(os.path.join(dashboard_directory, "SERVICEB")),
-                              call(os.path.join(dashboard_directory, "files/STORM.txt"))]
+                              call(os.path.join(dashboards_directory, "SERVICEB")),
+                              call(os.path.join(dashboards_directory, "files/STORM.txt"))]
 
   def _create_empty_options_mock(self):
     options = MagicMock()

ambari-server/src/test/resources/dashboards/README.txt

@@ -0,0 +1,18 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+Test directory for sync'ing a components custom folder - AMBARI-19018

ambari-server/src/test/resources/stacks/HDP/0.1/services/HDFS/metainfo.xml

@@ -57,6 +57,7 @@
           <name>DATANODE</name>
           <category>SLAVE</category>
           <cardinality>1+</cardinality>
+          <customFolder>dashboards</customFolder>
           <commandScript>
             <script>scripts/datanode.py</script>
             <scriptType>PYTHON</scriptType>