AMBARI-12776. HCSF service type support (Vijay Srinivasaraghavan via smohanty)

ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java

@@ -99,6 +99,9 @@ public class ExecutionCommand extends AgentCommand {
   @SerializedName("serviceName")
   private String serviceName;
 
+  @SerializedName("serviceType")
+  private String serviceType;  
+  
   @SerializedName("componentName")
   private String componentName;
 
@@ -270,6 +273,14 @@ public class ExecutionCommand extends AgentCommand {
   public void setServiceName(String serviceName) {
     this.serviceName = serviceName;
   }
+  
+  public String getServiceType() {
+	return serviceType;
+  }
+
+  public void setServiceType(String serviceType) {
+	this.serviceType = serviceType;
+  }
 
   public String getComponentName() {
     return componentName;
@@ -320,6 +331,7 @@ public class ExecutionCommand extends AgentCommand {
     String SERVICE_PACKAGE_FOLDER = "service_package_folder";
     String HOOKS_FOLDER = "hooks_folder";
     String STACK_NAME = "stack_name";
+    String SERVICE_TYPE = "service_type";
     String STACK_VERSION = "stack_version";
     String SERVICE_REPO_INFO = "service_repo_info";
     String PACKAGE_LIST = "package_list";

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java

@@ -1883,6 +1883,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
       }
     }
 
+    LOG.info("Adding service type info in createHostAction:: " + serviceInfo.getServiceType());
+    execCmd.setServiceType(serviceInfo.getServiceType());
+    
     execCmd.setConfigurations(configurations);
     execCmd.setConfigurationAttributes(configurationAttributes);
     execCmd.setConfigurationTags(configTags);

ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java

@@ -33,6 +33,7 @@ public class StackServiceResponse {
   private String stackName;
   private String stackVersion;
   private String serviceName;
+  private String serviceType;
   private String serviceDisplayName;
   private String userName;
   private String comments;
@@ -61,6 +62,7 @@ public class StackServiceResponse {
    */
   public StackServiceResponse(ServiceInfo service) {
     serviceName = service.getName();
+    serviceType = service.getServiceType();
     serviceDisplayName = service.getDisplayName();
     userName = null;
     comments = service.getComment();
@@ -107,8 +109,16 @@ public class StackServiceResponse {
   public void setServiceName(String serviceName) {
     this.serviceName = serviceName;
   }
+  
+  public String getServiceType() {
+	return serviceType;
+  }
+
+  public void setServiceType(String serviceType) {
+	this.serviceType = serviceType;
+  }
 
-  public String getServiceDisplayName() {
+public String getServiceDisplayName() {
     return serviceDisplayName;
   }
 

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java

@@ -40,6 +40,9 @@ public class StackServiceResourceProvider extends ReadOnlyResourceProvider {
 
   protected static final String SERVICE_NAME_PROPERTY_ID = PropertyHelper.getPropertyId(
       "StackServices", "service_name");
+  
+  protected static final String SERVICE_TYPE_PROPERTY_ID = PropertyHelper.getPropertyId(
+		  "StackServices", "service_type"); 
 
   public static final String STACK_NAME_PROPERTY_ID = PropertyHelper.getPropertyId(
       "StackServices", "stack_name");
@@ -124,6 +127,9 @@ public class StackServiceResourceProvider extends ReadOnlyResourceProvider {
 
       setResourceProperty(resource, SERVICE_NAME_PROPERTY_ID,
           response.getServiceName(), requestedIds);
+      
+      setResourceProperty(resource, SERVICE_TYPE_PROPERTY_ID,
+    		  response.getServiceType(), requestedIds); 
 
       setResourceProperty(resource, SERVICE_DISPLAY_NAME_PROPERTY_ID,
           response.getServiceDisplayName(), requestedIds);

ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java

@@ -56,6 +56,7 @@ public class ServiceInfo implements Validable{
   private String displayName;
   private String version;
   private String comment;
+  private String serviceType;
   private List<PropertyInfo> properties;
 
   @XmlElementWrapper(name="components")
@@ -253,8 +254,16 @@ public class ServiceInfo implements Validable{
   public void setDisplayName(String displayName) {
     this.displayName = displayName;
   }
+  
+  public String getServiceType() {
+	return serviceType;
+  }
+
+  public void setServiceType(String serviceType) {
+	this.serviceType = serviceType;
+  }
 
-  public String getVersion() {
+public String getVersion() {
     return version;
   }
 
@@ -345,6 +354,8 @@ public class ServiceInfo implements Validable{
     StringBuilder sb = new StringBuilder();
     sb.append("Service name:");
     sb.append(name);
+    sb.append("\nService type:");
+    sb.append(serviceType); 
     sb.append("\nversion:");
     sb.append(version);
     sb.append("\ncomment:");

ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java

@@ -2390,7 +2390,7 @@ public class ClusterImpl implements Cluster {
                 "single API request isn't supported. Conflicting services %s and %s for %s",
                                          serviceName, entry.getKey(), config.getType());
             IllegalArgumentException exception = new IllegalArgumentException(error);
-            LOG.error(error + ", config version not created");
+            LOG.error(error + ", config version not created for {}", serviceName);
             throw exception;
           } else {
             break;

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py

@@ -29,6 +29,9 @@ from resource_management.core.system import System
 from ambari_commons.os_check import OSCheck
 
 config = Script.get_config()
+
+service_type = default("serviceType","")
+
 sudo = AMBARI_SUDO_BINARY
 
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py

@@ -45,7 +45,8 @@ def setup_hdp_install_directory():
 def setup_config():
   import params
   stackversion = params.stack_version_unformatted
-  if params.has_namenode or stackversion.find('Gluster') >= 0:
+  Logger.info("Service Type: {0}".format(params.service_type))
+  if params.has_namenode or stackversion.find('Gluster') >= 0 or params.service_type == 'HCFS':
     # create core-site only if the hadoop config diretory exists
     XmlConfig("core-site.xml",
               conf_dir=params.hadoop_conf_dir,

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py

@@ -37,6 +37,8 @@ from ambari_commons.constants import AMBARI_SUDO_BINARY
 config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
+service_type = default("serviceType","")
+
 artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
 jdk_name = default("/hostLevelParams/jdk_name", None)
 java_home = config['hostLevelParams']['java_home']

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py

@@ -132,7 +132,8 @@ def set_uid(user, user_dirs):
 def setup_hadoop_env():
   import params
   stackversion = params.stack_version_unformatted
-  if params.has_namenode or stackversion.find('Gluster') >= 0:
+  Logger.info("Service Type: {0}".format(params.service_type))
+  if params.has_namenode or stackversion.find('Gluster') >= 0 or params.service_type == 'HCFS':
     if params.security_enabled:
       tc_owner = "root"
     else:

ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java

@@ -170,7 +170,7 @@ public class AmbariManagementControllerTest {
   private static final String PROPERTY_NAME = "hbase.regionserver.msginterval";
   private static final String SERVICE_NAME = "HDFS";
   private static final String FAKE_SERVICE_NAME = "FAKENAGIOS";
-  private static final int STACK_VERSIONS_CNT = 14;
+  private static final int STACK_VERSIONS_CNT = 15;
   private static final int REPOS_CNT = 3;
   private static final int STACKS_CNT = 3;
   private static final int STACK_PROPERTIES_CNT = 103;

ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java

@@ -104,18 +104,29 @@ public class StackManagerTest {
   @Test
   public void testGetsStacks() throws Exception {
     Collection<StackInfo> stacks = stackManager.getStacks();
-    assertEquals(18, stacks.size());
+    assertEquals(19, stacks.size());
   }
 
   @Test
   public void testGetStacksByName() {
     Collection<StackInfo> stacks = stackManager.getStacks("HDP");
-    assertEquals(14, stacks.size());
+    assertEquals(15, stacks.size());
 
     stacks = stackManager.getStacks("OTHER");
     assertEquals(2, stacks.size());
   }
 
+  @Test
+  public void testHCFSServiceType() {
+    
+    StackInfo stack = stackManager.getStack("HDP", "2.2.0.ECS");
+    ServiceInfo service = stack.getService("ECS");
+    assertEquals(service.getServiceType(),"HCFS");
+    
+    service = stack.getService("HDFS");
+    assertNull(service);
+  }  
+
   @Test
   public void testGetStack() {
     StackInfo stack = stackManager.getStack("HDP", "0.1");

ambari-server/src/test/java/org/apache/ambari/server/state/ServiceInfoTest.java

@@ -62,6 +62,13 @@ public class ServiceInfoTest {
         "      <comment>Apache Hadoop Distributed File System</comment>\n" +
         "      <version>2.1.0.2.0</version>\n" +
         "    </service>\n" +
+        "    <service>\n" +
+        "      <name>HCFS_SERVICE</name>\n" +
+        "      <displayName>HCFS_SERVICE</displayName>\n" +
+        "      <comment>Hadoop Compatible File System</comment>\n" +
+        "      <version>2.1.1.0</version>\n" +
+        "      <serviceType>HCFS</serviceType>\n" +
+        "    </service>\n" +
         "  </services>\n" +
         "</metainfo>\n";
     
@@ -70,6 +77,7 @@ public class ServiceInfoTest {
     assertTrue(serviceInfoMap.get("RESTART").isRestartRequiredAfterRackChange());
     assertFalse(serviceInfoMap.get("NO_RESTART").isRestartRequiredAfterRackChange());
     assertNull(serviceInfoMap.get("DEFAULT_RESTART").isRestartRequiredAfterRackChange());
+    assertEquals(serviceInfoMap.get("HCFS_SERVICE").getServiceType(),"HCFS");
   }
 
   @Test

ambari-server/src/test/resources/stacks/HDP/2.2.0.ECS/metainfo.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+    <versions>
+      <active>true</active>
+    </versions>
+    <extends>2.2.0</extends>
+</metainfo>
+

ambari-server/src/test/resources/stacks/HDP/2.2.0.ECS/repos/hdp.json

@@ -0,0 +1,10 @@
+{
+  "HDP-2.2.0": {
+    "latest": {
+      "centos6": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.2.0.0-123",
+      "redhat6": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.2.2.0-123",
+      "oraclelinux6": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.2.0.0-123",
+      "suse11": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/suse11/2.x/BUILDS/2.2.0.0-123/hdp.repo"
+    }
+  }
+}

ambari-server/src/test/resources/stacks/HDP/2.2.0.ECS/repos/repoinfo.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<reposinfo>
+  <latest>./hdp.json</latest>
+  <os family="centos6">
+    <repo>
+      <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.0.0</baseurl>
+      <repoid>HDP-2.2.0</repoid>
+      <reponame>HDP</reponame>
+    </repo>
+  </os>
+  <os family="centos5">
+    <repo>
+      <baseurl>http://public-repo-1.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0</baseurl>
+      <repoid>HDP-2.2.0</repoid>
+      <reponame>HDP</reponame>
+    </repo>
+  </os>
+  <os family="redhat6">
+    <repo>
+      <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.0.0</baseurl>
+      <repoid>HDP-2.2.0</repoid>
+      <reponame>HDP</reponame>
+    </repo>
+  </os>
+  <os family="redhat5">
+    <repo>
+      <baseurl>http://public-repo-1.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0</baseurl>
+      <repoid>HDP-2.2.0</repoid>
+      <reponame>HDP</reponame>
+    </repo>
+  </os>
+  <os family="suse11">
+    <repo>
+      <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11/2.x/updates/2.2.0.0</baseurl>
+      <repoid>HDP-2.2.0</repoid>
+      <reponame>HDP</reponame>
+    </repo>
+  </os>
+  <os family="any">
+    <repo>
+      <baseurl>http://public-repo.emc.com/ECS/2.2/updates/2.2.0.0</baseurl>
+      <repoid>ECS-2.2.0.0</repoid>
+      <reponame>ECS</reponame>
+    </repo>
+  </os>
+</reposinfo>

ambari-server/src/test/resources/stacks/HDP/2.2.0.ECS/services/ECS/metainfo.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>ECS</name>
+      <displayName>ECS</displayName>
+      <serviceType>HCFS</serviceType>
+      <comment>An Hadoop Compatible File System</comment>
+      <version>2.2.0.0</version>
+      <components>
+        <component>
+          <name>ECS_CLIENT</name>
+          <category>CLIENT</category>
+        </component>
+      </components>
+    </service>
+  </services>
+</metainfo>

ambari-server/src/test/resources/stacks/HDP/2.2.0.ECS/services/HDFS/metainfo.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>HDFS</name>
+      <comment>Apache Hadoop Distributed File System</comment>
+      <version>2.1.0.2.0.6.0</version>
+      <deleted>true</deleted>
+    </service>
+  </services>
+</metainfo>

ambari-web/app/controllers/wizard/step4_controller.js

@@ -261,7 +261,7 @@ App.WizardStep4Controller = Em.ArrayController.extend({
     var dfsServices = ['HDFS', 'GLUSTERFS'];
     var availableServices = this.filterProperty('isInstalled',false);
     availableServices.forEach(function(service){
-      if (dfsServices.contains(service.get('serviceName'))) {
+      if (dfsServices.contains(service.get('serviceName')) || service.get('serviceType') == 'HCFS' ) {
         console.log("found DFS " + service.get('serviceName'));
         bDFSStack=true;
       }

ambari-web/app/mappers/stack_service_mapper.js

@@ -25,6 +25,7 @@ App.stackServiceMapper = App.QuickDataMapper.create({
     id: 'service_name',
     stack_id: 'stack_id',
     service_name: 'service_name',
+    service_type: 'service_type',
     display_name: 'display_name',
     config_types: 'config_types',
     comments: 'comments',

ambari-web/app/models/stack_service.js

@@ -27,6 +27,7 @@ require('models/configs/objects/service_config_category');
  */
 App.StackService = DS.Model.extend({
   serviceName: DS.attr('string'),
+  serviceType: DS.attr('string'),
   displayName: DS.attr('string'),
   comments: DS.attr('string'),
   configTypes: DS.attr('object'),
@@ -53,7 +54,10 @@ App.StackService = DS.Model.extend({
   // Is the service a distributed filesystem
   isDFS: function () {
     var dfsServices = ['HDFS', 'GLUSTERFS'];
-    return dfsServices.contains(this.get('serviceName'));
+    if( this.get('serviceType') == 'HCFS' || dfsServices.contains(this.get('serviceName')) )
+    	return true;
+    else
+    	return false;
   }.property('serviceName'),
 
   // Primary DFS. used if there is more than one DFS in a stack.

dev-support/docker/docker/Dockerfile

@@ -16,7 +16,7 @@ RUN echo root:changeme | chpasswd
 
 ## Install some basic utilities that aren't in the default image
 RUN yum clean all -y && yum update -y
-RUN yum -y install vim wget rpm-build sudo which telnet tar openssh-server openssh-clients ntp git python-setuptools httpd
+RUN yum -y install vim wget rpm-build sudo which telnet tar openssh-server openssh-clients ntp git python-setuptools python-devel httpd
 # phantomjs dependency
 RUN yum -y install fontconfig freetype libfreetype.so.6 libfontconfig.so.1 libstdc++.so.6
 RUN rpm -e --nodeps --justdb glibc-common
@@ -25,11 +25,8 @@ RUN yum -y install glibc-common
 ENV HOME /root
 
 #Install JAVA
-# RUN wget --no-check-certificate --no-cookies --header "Cookie:oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/7u55-b13/jdk-7u55-linux-x64.rpm -O jdk-7u55-linux-x64.rpm
-# RUN yum -y install jdk-7u55-linux-x64.rpm
-RUN wget --no-check-certificate --no-cookies --header "Cookie:oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/6u45-b06/jdk-6u45-linux-x64-rpm.bin -O jdk-6u45-linux-x64-rpm.bin
-RUN chmod +x jdk-6u45-linux-x64-rpm.bin
-RUN ./jdk-6u45-linux-x64-rpm.bin
+RUN wget --no-check-certificate --no-cookies --header "Cookie:oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/7u55-b13/jdk-7u55-linux-x64.rpm -O jdk-7u55-linux-x64.rpm
+RUN rpm -ivh jdk-7u55-linux-x64.rpm
 ENV JAVA_HOME /usr/java/default/
 
 #Install Maven
@@ -50,6 +47,9 @@ RUN cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
 RUN chmod 600 /root/.ssh/authorized_keys
 RUN sed -ri 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config
 
+#To allow bower install behind proxy. See https://github.com/bower/bower/issues/731
+RUN git config --global url."https://".insteadOf git://
+
 # Install python, nodejs and npm
 RUN yum -y install http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
 RUN yum -y install nodejs npm --enablerepo=epel
@@ -59,8 +59,8 @@ RUN npm install -g brunch@1.7.13
 WORKDIR /tmp
 RUN git clone https://github.com/apache/ambari.git
 WORKDIR /tmp/ambari
-RUN mvn versions:set -DnewVersion=1.6.1.0
-RUN mvn -B clean install package rpm:rpm -DskipTests -DnewVersion=1.6.1.0 -Dpython.ver="python >= 2.6" -Preplaceurl
+
+RUN mvn -B -X clean install package rpm:rpm -DskipTests -Dpython.ver="python >= 2.6" -Preplaceurl
 
 # clean git code because I want to use the one on local filesystem.
 WORKDIR /tmp