AMBARI-8676: Common Services: Service package dir is set to path relative to stackRoot and hence wouldn't work when service package dir is in common-services (Jayush Luniya)

ambari-agent/src/main/python/ambari_agent/FileCache.py

@@ -67,8 +67,7 @@ class FileCache():
     Returns a base directory for service
     """
     service_subpath = command['commandParams']['service_package_folder']
-    subpath = os.path.join(self.STACKS_CACHE_DIRECTORY, service_subpath)
-    return self.provide_directory(self.cache_dir, subpath,
+    return self.provide_directory(self.cache_dir, service_subpath,
                                   server_url_prefix)
 
 

ambari-agent/src/test/python/ambari_agent/TestFileCache.py

@@ -64,7 +64,7 @@ class TestFileCache(TestCase):
     fileCache = FileCache(self.config)
     command = {
       'commandParams' : {
-        'service_package_folder' : 'HDP/2.1.1/services/ZOOKEEPER/package'
+        'service_package_folder' : 'stacks/HDP/2.1.1/services/ZOOKEEPER/package'
       }
     }
     res = fileCache.get_service_base_dir(command, "server_url_pref")

ambari-server/src/main/java/org/apache/ambari/server/stack/CommonServiceDirectory.java

@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.stack;
+
+import org.apache.ambari.server.AmbariException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+/**
+ * Encapsulates IO operations on a common services directory.
+ */
+public class CommonServiceDirectory extends ServiceDirectory {
+  /**
+   * logger instance
+   */
+  private static final Logger LOG = LoggerFactory.getLogger(CommonServiceDirectory.class);
+
+  /**
+   * Constructor.
+   *
+   * @param servicePath     path of the service directory
+   * @throws org.apache.ambari.server.AmbariException if unable to parse the service directory
+   */
+  public CommonServiceDirectory(String servicePath) throws AmbariException {
+    super(servicePath);
+  }
+
+  @Override
+  /**
+   * Parse common service directory
+   * packageDir Format: common-services/<serviceName>/<serviceVersion>/package
+   * Example:
+   *  directory: "/var/lib/ambari-server/resources/common-services/HDFS/1.0"
+   *  packageDir: "common-services/HDFS/1.0/package"
+   *
+   * @throws AmbariException
+   */
+  protected void parsePath() throws AmbariException {
+    File serviceVersionDir = new File(getAbsolutePath());
+    File serviceDir = serviceVersionDir.getParentFile();
+
+    String serviceId = String.format("%s/%s", serviceDir.getName(), serviceVersionDir.getName());
+
+    File absPackageDir = new File(getAbsolutePath() + File.separator + PACKAGE_FOLDER_NAME);
+    if(absPackageDir.isDirectory()) {
+      packageDir = absPackageDir.getPath().substring(serviceDir.getParentFile().getParentFile().getPath().length() + 1);
+      LOG.debug(String.format("Service package folder for common service %s has been resolved to %s",
+          serviceId, packageDir));
+    } else {
+      LOG.debug(String.format("Service package folder %s for common service %s does not exist.",
+          absPackageDir, serviceId ));
+    }
+    parseMetaInfoFile();
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java

@@ -30,8 +30,7 @@ import java.io.File;
 /**
  * Encapsulates IO operations on a stack definition service directory.
  */
-public class ServiceDirectory extends StackDefinitionDirectory {
-
+public abstract class ServiceDirectory extends StackDefinitionDirectory {
   /**
    * metrics file
    */
@@ -50,7 +49,7 @@ public class ServiceDirectory extends StackDefinitionDirectory {
   /**
    * package directory path
    */
-  private String packageDir;
+  protected String packageDir;
 
   /**
    * service metainfo file object representation
@@ -65,7 +64,7 @@ public class ServiceDirectory extends StackDefinitionDirectory {
   /**
    * package directory name
    */
-  private static final String PACKAGE_FOLDER_NAME = "package";
+  protected static final String PACKAGE_FOLDER_NAME = "package";
 
   /**
    * service metainfo file name
@@ -86,7 +85,7 @@ public class ServiceDirectory extends StackDefinitionDirectory {
   /**
    * Constructor.
    *
-   * @param servicePath  path of the service directory
+   * @param servicePath     path of the service directory
    * @throws AmbariException if unable to parse the service directory
    */
   public ServiceDirectory(String servicePath) throws AmbariException {
@@ -154,29 +153,8 @@ public class ServiceDirectory extends StackDefinitionDirectory {
 
   /**
    * Parse the service directory.
-   *
-   * @throws AmbariException if unable to parse the service directory
    */
-  private void parsePath() throws AmbariException {
-
-    File serviceDir = new File(getAbsolutePath());
-    File stackVersionDir = serviceDir.getParentFile().getParentFile();
-    File stackDir = stackVersionDir.getParentFile();
-
-    String stackId = String.format("%s-%s", stackDir.getName(), stackVersionDir.getName());
-
-    File absPackageDir = new File(getAbsolutePath() + File.separator + PACKAGE_FOLDER_NAME);
-    if (absPackageDir.isDirectory()) {
-      packageDir = absPackageDir.getPath().substring(stackDir.getParentFile().getPath().length() + 1);
-      LOG.debug(String.format("Service package folder for service %s for stack %s has been resolved to %s",
-          serviceDir.getName(), stackId, packageDir));
-    } else {
-      //todo: this seems like it should be an error case
-      LOG.debug(String.format("Service package folder %s for service %s for stack %s does not exist.",
-          absPackageDir, serviceDir.getName(), stackId));
-    }
-    parseMetaInfoFile();
-  }
+  protected abstract void parsePath() throws AmbariException;
 
   /**
    * Unmarshal the metainfo file into its object representation.
@@ -184,7 +162,7 @@ public class ServiceDirectory extends StackDefinitionDirectory {
    * @throws AmbariException if the metainfo file doesn't exist or
    *                         unable to unmarshal the metainfo file
    */
-  private void parseMetaInfoFile() throws AmbariException {
+  protected void parseMetaInfoFile() throws AmbariException {
     File f = new File(getAbsolutePath() + File.separator + SERVICE_METAINFO_FILE_NAME);
     if (! f.exists()) {
       throw new AmbariException(String.format("Stack Definition Service at '%s' doesn't contain a metainfo.xml file",

ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java

@@ -357,7 +357,7 @@ public class StackDirectory extends StackDefinitionDirectory {
         for (File d : serviceFolders) {
           if (d.isDirectory()) {
             try {
-              dirs.add(new ServiceDirectory(d.getAbsolutePath()));
+              dirs.add(new StackServiceDirectory(d.getAbsolutePath()));
             } catch (AmbariException e) {
               //todo: this seems as though we should propagate this exception
               //todo: eating it now to keep backwards compatibility

ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java

@@ -245,7 +245,7 @@ public class StackManager {
         for (File serviceFolder : commonService.listFiles(AmbariMetaInfo.FILENAME_FILTER)) {
           String serviceName = serviceFolder.getParentFile().getName();
           String serviceVersion = serviceFolder.getName();
-          ServiceDirectory serviceDirectory = new ServiceDirectory(serviceFolder.getPath());
+          ServiceDirectory serviceDirectory = new CommonServiceDirectory(serviceFolder.getPath());
           ServiceMetainfoXml metaInfoXml = serviceDirectory.getMetaInfoFile();
           for (ServiceInfo serviceInfo : metaInfoXml.getServices()) {
             ServiceModule serviceModule = new ServiceModule(stackContext, serviceInfo, serviceDirectory, true);

ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java

@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.stack;
+
+import org.apache.ambari.server.AmbariException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+
+/**
+ * Encapsulates IO operations on a stack service directory.
+ */
+public class StackServiceDirectory extends ServiceDirectory {
+
+  /**
+   * logger instance
+   */
+  private static final Logger LOG = LoggerFactory.getLogger(StackServiceDirectory.class);
+
+  /**
+   * Constructor.
+   *
+   * @param servicePath     path of the service directory
+   * @throws org.apache.ambari.server.AmbariException if unable to parse the service directory
+   */
+  public StackServiceDirectory(String servicePath) throws AmbariException {
+    super(servicePath);
+  }
+
+  @Override
+  /**
+   * Parse stack service directory.
+   * packageDir Format: stacks/<stackName>/<stackVersion>/services/<serviceName>/package
+   * Example:
+   *  directory: "/var/lib/ambari-server/resources/stacks/HDP/2.0.6/services/HDFS"
+   *  packageDir: "stacks/HDP/2.0.6/services/HDFS/package"
+   * @throws AmbariException if unable to parse the service directory
+   */
+  protected void parsePath() throws AmbariException {
+    File serviceDir = new File(getAbsolutePath());
+    File stackVersionDir = serviceDir.getParentFile().getParentFile();
+    File stackDir = stackVersionDir.getParentFile();
+
+    String stackId = String.format("%s-%s", stackDir.getName(), stackVersionDir.getName());
+
+    File absPackageDir = new File(getAbsolutePath() + File.separator + PACKAGE_FOLDER_NAME);
+    if (absPackageDir.isDirectory()) {
+      packageDir = absPackageDir.getPath().substring(stackDir.getParentFile().getParentFile().getPath().length() + 1);
+      LOG.debug(String.format("Service package folder for service %s for stack %s has been resolved to %s",
+          serviceDir.getName(), stackId, packageDir));
+    } else {
+      LOG.debug(String.format("Service package folder %s for service %s for stack %s does not exist.",
+          absPackageDir, serviceDir.getName(), stackId));
+    }
+    parseMetaInfoFile();
+  }
+}

ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java

@@ -81,6 +81,7 @@ import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
 import org.apache.ambari.server.state.stack.MetricDefinition;
 import org.apache.ambari.server.state.stack.OsFamily;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -1249,12 +1250,10 @@ public class AmbariMetaInfoTest {
 
   @Test
   public void testServicePackageDirInheritance() throws Exception {
-    String assertionTemplate07 = "HDP/2.0.7/services/%s/package";
-    String assertionTemplate08 = "HDP/2.0.8/services/%s/package";
-    if (System.getProperty("os.name").contains("Windows")) {
-      assertionTemplate07 = "HDP\\2.0.7\\services\\%s\\package";
-      assertionTemplate08 = "HDP\\2.0.8\\services\\%s\\package";
-    }
+    String assertionTemplate07 = StringUtils.join(
+        new String[]{"stacks", "HDP", "2.0.7", "services", "%s", "package"}, File.separator);
+    String assertionTemplate08 = StringUtils.join(
+        new String[]{"stacks", "HDP", "2.0.8", "services", "%s", "package"}, File.separator);
     // Test service package dir determination in parent
     ServiceInfo service = metaInfo.getService(STACK_NAME_HDP, "2.0.7", "HBASE");
     Assert.assertEquals(String.format(assertionTemplate07, "HBASE"),

ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java

@@ -23,6 +23,7 @@ import org.apache.ambari.server.metadata.ActionMetadata;
 import org.apache.ambari.server.orm.dao.MetainfoDAO;
 import org.apache.ambari.server.state.*;
 import org.apache.ambari.server.state.stack.OsFamily;
+import org.apache.commons.lang.StringUtils;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -164,4 +165,29 @@ public class StackManagerCommonServicesTest {
 
     assertEquals(pigService.getParent(), "common-services/PIG/1.0");
   }
+
+  @Test
+  public void testGetServicePackageFolder() {
+    StackInfo stack = stackManager.getStack("HDP", "0.1");
+    assertNotNull(stack);
+    assertEquals("HDP", stack.getName());
+    assertEquals("0.1", stack.getVersion());
+    ServiceInfo hdfsService1 = stack.getService("HDFS");
+    assertNotNull(hdfsService1);
+
+    stack = stackManager.getStack("HDP", "0.2");
+    assertNotNull(stack);
+    assertEquals("HDP", stack.getName());
+    assertEquals("0.2", stack.getVersion());
+    ServiceInfo hdfsService2 = stack.getService("HDFS");
+    assertNotNull(hdfsService2);
+
+    String packageDir1 = StringUtils.join(
+        new String[]{"common-services", "HDFS", "1.0", "package"}, File.separator);
+    String packageDir2 = StringUtils.join(
+        new String[]{"stacks_with_common_services", "HDP", "0.2", "services", "HDFS", "package"}, File.separator);
+
+    assertEquals(packageDir1, hdfsService1.getServicePackageFolder());
+    assertEquals(packageDir2, hdfsService2.getServicePackageFolder());
+  }
 }

ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java

@@ -32,6 +32,7 @@ import static org.junit.Assert.fail;
 import java.io.File;
 import java.util.*;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.metadata.ActionMetadata;
@@ -292,12 +293,10 @@ public class StackManagerTest {
     // overridden value
     assertEquals("Apache Hadoop Stream processing framework (Extended)", stormService.getComment());
     assertEquals("New version", stormService.getVersion());
-    if (System.getProperty("os.name").contains("Windows")) {
-      assertEquals("OTHER\\1.0\\services\\STORM\\package", stormService.getServicePackageFolder());
-    }
-    else {
-      assertEquals("OTHER/1.0/services/STORM/package", stormService.getServicePackageFolder());
-    }
+    String packageDir = StringUtils.join(
+        new String[]{"stacks", "OTHER", "1.0", "services", "STORM", "package"}, File.separator);
+    assertEquals(packageDir, stormService.getServicePackageFolder());
+
     // compare components
     List<ComponentInfo> stormServiceComponents = stormService.getComponents();
     List<ComponentInfo> baseStormServiceComponents = baseStormService.getComponents();

ambari-server/src/test/resources/common-services/HDFS/1.0/package/dummy-script.py

@@ -0,0 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""

ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/HDFS/package/dummy-script.py

@@ -0,0 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""