AMBARI-15007. Make amazon2015 to be part of redhat6 family (aonishuk)

ambari-agent/pom.xml

@@ -85,24 +85,6 @@
         <artifactId>maven-compiler-plugin</artifactId>
         <version>3.0</version>
       </plugin>
-      <plugin>
-        <artifactId>maven-assembly-plugin</artifactId>
-        <configuration>
-          <tarLongFileMode>gnu</tarLongFileMode>
-          <descriptors>
-            <descriptor>src/packages/tarball/all.xml</descriptor>
-          </descriptors>
-        </configuration>
-        <executions>
-          <execution>
-            <id>build-tarball</id>
-            <phase>prepare-package</phase>
-            <goals>
-              <goal>single</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
@@ -274,6 +256,24 @@
           </dataSet>
         </configuration>
       </plugin>
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <configuration>
+          <tarLongFileMode>gnu</tarLongFileMode>
+          <descriptors>
+            <descriptor>src/packages/tarball/all.xml</descriptor>
+          </descriptors>
+        </configuration>
+        <executions>
+          <execution>
+            <id>build-tarball</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <artifactId>maven-resources-plugin</artifactId>
         <version>2.6</version>

ambari-common/src/main/python/ambari_commons/os_check.py

@@ -56,6 +56,8 @@ RESOURCES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "resou
 
 # family JSON data
 OSFAMILY_JSON_RESOURCE = "os_family.json"
+JSON_OS_MAPPING = "mapping"
+JSON_OS_ALIASES = "aliases"
 JSON_OS_TYPE = "distro"
 JSON_OS_VERSION = "versions"
 JSON_EXTENDS = "extends"
@@ -76,6 +78,8 @@ VER_NT_SERVER = 3
 _IS_ORACLE_LINUX = os.path.exists('/etc/oracle-release')
 _IS_REDHAT_LINUX = os.path.exists('/etc/redhat-release')
 
+SYSTEM_RELEASE_FILE = "/etc/system-release"
+
 def _is_oracle_linux():
   return _IS_ORACLE_LINUX
 
@@ -84,16 +88,16 @@ def _is_redhat_linux():
 
 def advanced_check(distribution):
   distribution = list(distribution)
-  if os.path.exists("/etc/issue"):
-    with open("/etc/issue", "rb") as fp:
+  if os.path.exists(SYSTEM_RELEASE_FILE):
+    with open(SYSTEM_RELEASE_FILE, "rb") as fp:
       issue_content = fp.read()
   
     if "Amazon" in issue_content:
       distribution[0] = "amazon"
-      search_groups = re.search('(\d+)\.(\d+)', issue_content)
+      search_groups = re.search('(\d+\.\d+)', issue_content)
       
       if search_groups:
-        distribution[1] = search_groups.group(1) # if version is 2015.09 only get 2015.
+        distribution[1] = search_groups.group(1)
       
   return tuple(distribution)
     
@@ -114,16 +118,24 @@ class OS_CONST_TYPE(type):
       f = open(os.path.join(RESOURCES_DIR, OSFAMILY_JSON_RESOURCE))
       json_data = eval(f.read())
       f.close()
-      for family in json_data:
+      
+      if JSON_OS_MAPPING not in json_data:
+        raise Exception("Invalid {0}".format(OSFAMILY_JSON_RESOURCE))
+      
+      json_mapping_data = json_data[JSON_OS_MAPPING]
+      
+      for family in json_mapping_data:
         cls.FAMILY_COLLECTION += [family]
-        cls.OS_COLLECTION += json_data[family][JSON_OS_TYPE]
+        cls.OS_COLLECTION += json_mapping_data[family][JSON_OS_TYPE]
         cls.OS_FAMILY_COLLECTION += [{
           'name': family,
-          'os_list': json_data[family][JSON_OS_TYPE]
+          'os_list': json_mapping_data[family][JSON_OS_TYPE]
         }]
         
-        if JSON_EXTENDS in json_data[family]:
-          cls.OS_FAMILY_COLLECTION[-1][JSON_EXTENDS] = json_data[family][JSON_EXTENDS]
+        if JSON_EXTENDS in json_mapping_data[family]:
+          cls.OS_FAMILY_COLLECTION[-1][JSON_EXTENDS] = json_mapping_data[family][JSON_EXTENDS]
+          
+        cls.OS_TYPE_ALIASES = json_data[JSON_OS_ALIASES] if JSON_OS_ALIASES in json_data else {}
     except:
       raise Exception("Couldn't load '%s' file" % OSFAMILY_JSON_RESOURCE)
 
@@ -194,7 +206,24 @@ class OSCheck:
         distribution = ("Darwin", "TestOnly", "1.1.1", "1.1.1", "1.1")
     
     return distribution
-
+  
+  @staticmethod
+  def get_alias(os_type, os_version):
+    version_parts = os_version.split('.')
+    full_os_and_major_version = os_type + version_parts[0]
+
+    if full_os_and_major_version in OSConst.OS_TYPE_ALIASES:
+      alias = OSConst.OS_TYPE_ALIASES[full_os_and_major_version]
+      re_groups = re.search('(\D+)(\d+)$', alias).groups()
+      os_type = re_groups[0]
+      os_major_version = re_groups[1]
+      
+      version_parts[0] = os_major_version
+      os_version = '.'.join(version_parts)
+      
+    return os_type, os_version
+      
+    
   @staticmethod
   def get_os_type():
     """
@@ -205,6 +234,10 @@ class OSCheck:
 
     In case cannot detect - exit.
     """
+    return OSCheck.get_alias(OSCheck._get_os_type(), OSCheck._get_os_version())[0]
+
+  @staticmethod
+  def _get_os_type():
     # Read content from /etc/*-release file
     # Full release name
     dist = OSCheck.os_distribution()
@@ -212,18 +245,18 @@ class OSCheck:
 
     # special cases
     if _is_oracle_linux():
-      return 'oraclelinux'
+      operatingSystem = 'oraclelinux'
     elif operatingSystem.startswith('suse linux enterprise server'):
-      return 'sles'
+      operatingSystem = 'sles'
     elif operatingSystem.startswith('red hat enterprise linux'):
-      return 'redhat'
+      operatingSystem = 'redhat'
     elif operatingSystem.startswith('darwin'):
-      return 'mac'
+      operatingSystem = 'mac'
 
-    if operatingSystem != '':
-      return operatingSystem
-    else:
+    if operatingSystem == '':
       raise Exception("Cannot detect os type. Exiting...")
+    
+    return operatingSystem
 
   @staticmethod
   def get_os_family():
@@ -257,11 +290,15 @@ class OSCheck:
 
     In case cannot detect raises exception.
     """
+    return OSCheck.get_alias(OSCheck._get_os_type(), OSCheck._get_os_version())[1]
+    
+  @staticmethod
+  def _get_os_version():
     # Read content from /etc/*-release file
     # Full release name
     dist = OSCheck.os_distribution()
     dist = dist[1]
-
+    
     if dist:
       return dist
     else:

ambari-common/src/main/python/ambari_commons/resources/os_family.json

@@ -1,72 +1,69 @@
 {
-  "redhat": {
-    "distro": [
-      "redhat",
-      "fedora",
-      "centos",
-      "oraclelinux",
-      "ascendos",
-      "xenserver",
-      "oel",
-      "ovs",
-      "cloudlinux",
-      "slc",
-      "scientific",
-      "psbm",
-      "centos linux"
-    ],
-    "versions": [
-      6,
-      7
-    ]
-  },
-  "amazon": {
-    "extends" : "redhat",
-    "distro": [
-      "amazon"
-    ],
-    "versions": [
-      2015
-    ]
-  },
-  "debian": {
-    "extends" : "ubuntu",
-    "distro": [
-      "debian"
-    ],
-    "versions": [
-      7
-    ]
-  },
-  "ubuntu": {
-    "distro": [
-      "ubuntu"
-    ],
-    "versions": [
-      12,
-      14
-    ]
-  },
-  "suse": {
-    "distro": [
-      "sles",
-      "sled",
-      "opensuse",
-      "suse"
-    ],
-    "versions": [
-      11
-    ]
-  },
-  "winsrv": {
-    "distro": [
-      "win2008server",
-      "win2008serverr2",
-      "win2012server",
-      "win2012serverr2"
-    ],
-    "versions": [
-      6
-    ]
-  }
+  "mapping": {
+      "redhat": {
+        "distro": [
+          "redhat",
+          "fedora",
+          "centos",
+          "oraclelinux",
+          "amazon",
+          "ascendos",
+          "xenserver",
+          "oel",
+          "ovs",
+          "cloudlinux",
+          "slc",
+          "scientific",
+          "psbm",
+          "centos linux"
+        ],
+        "versions": [
+          6,
+          7
+        ]
+      },
+      "debian": {
+        "extends" : "ubuntu",
+        "distro": [
+          "debian"
+        ],
+        "versions": [
+          7
+        ]
+      },
+      "ubuntu": {
+        "distro": [
+          "ubuntu"
+        ],
+        "versions": [
+          12,
+          14
+        ]
+      },
+      "suse": {
+        "distro": [
+          "sles",
+          "sled",
+          "opensuse",
+          "suse"
+        ],
+        "versions": [
+          11
+        ]
+      },
+      "winsrv": {
+        "distro": [
+          "win2008server",
+          "win2008serverr2",
+          "win2012server",
+          "win2012serverr2"
+        ],
+        "versions": [
+          6
+        ]
+      }
+    },
+    "aliases": {
+      "amazon2015": "amazon6"
+    }
 }

ambari-server/pom.xml

@@ -40,8 +40,8 @@
     <resourceManagementSrcLocation>${project.basedir}/../ambari-common/src/main/python/resource_management</resourceManagementSrcLocation>
     <customActionsRoot>src/main/resources/custom_actions</customActionsRoot>
     <ambariProperties>conf/unix/ambari.properties</ambariProperties>
-    <commonServicesSrcLocation>target/classes/common-services</commonServicesSrcLocation>
-    <stacksSrcLocation>target/classes/stacks/${stack.distribution}</stacksSrcLocation>
+    <commonServicesSrcLocation>src/main/resources/common-services</commonServicesSrcLocation>
+    <stacksSrcLocation>src/main/resources/stacks/${stack.distribution}</stacksSrcLocation>
     <tarballResourcesFolder>src/main/resources</tarballResourcesFolder>
     <skipPythonTests>false</skipPythonTests>
     <hadoop.version>2.7.1</hadoop.version>
@@ -562,14 +562,6 @@
           <exclude>common-services/**</exclude>
         </excludes>
       </resource>
-      <resource>
-        <directory>src/main/resources</directory>
-        <filtering>false</filtering>
-        <includes>
-          <include>stacks/**</include>
-          <include>common-services/**</include>
-        </includes>
-      </resource>
     </resources>
   </build>
   <profiles>

ambari-server/src/main/assemblies/server.xml

@@ -301,7 +301,7 @@
     </file>
     <file>
       <fileMode>755</fileMode>
-      <source>target/classes/stacks/stack_advisor.py</source>
+      <source>src/main/resources/stacks/stack_advisor.py</source>
       <outputDirectory>/var/lib/ambari-server/resources/stacks</outputDirectory>
     </file>
     <file>

ambari-server/src/main/java/org/apache/ambari/server/state/stack/JsonOsFamilyRoot.java

@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.stack;
+
+import java.util.Map;
+
+public class JsonOsFamilyRoot {
+  private Map<String, JsonOsFamilyEntry> mapping;
+  private Map<String, String> aliases;
+  
+  public Map<String, JsonOsFamilyEntry> getMapping() {
+    return mapping;
+  }
+  public void setMapping(Map<String, JsonOsFamilyEntry> mapping) {
+    this.mapping = mapping;
+  }
+  public Map<String, String> getAliases() {
+    return aliases;
+  }
+  public void setAliases(Map<String, String> aliases) {
+    this.aliases = aliases;
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java

@@ -48,11 +48,14 @@ public class OsFamily {
     private final String os_pattern = "([\\D]+|(?:[\\D]+[\\d]+[\\D]+))([\\d]*)";
     private final String OS_DISTRO = "distro";
     private final String OS_VERSION = "versions";
+    private final String OS_MAPPING = "mapping";
+    private final String OS_ALIASES = "aliases";
     private final String LOAD_CONFIG_MSG = "Could not load OS family definition from %s file";
     private final String FILE_NAME = "os_family.json";
     private final Logger LOG = LoggerFactory.getLogger(OsFamily.class);
 
     private Map<String, JsonOsFamilyEntry> osMap = null;
+    private JsonOsFamilyRoot jsonOsFamily = null;
 
   /**
    * Initialize object
@@ -77,9 +80,10 @@ public class OsFamily {
         if (!f.exists()) throw new Exception();
         inputStream = new FileInputStream(f);
 
-        Type type = new TypeToken<Map<String, JsonOsFamilyEntry>>() {}.getType();
+        Type type = new TypeToken<JsonOsFamilyRoot>() {}.getType();
         Gson gson = new Gson();
-        osMap = gson.fromJson(new InputStreamReader(inputStream), type);
+        jsonOsFamily = gson.fromJson(new InputStreamReader(inputStream), type);
+        osMap = jsonOsFamily.getMapping();
       } catch (Exception e) {
         LOG.error(String.format(LOAD_CONFIG_MSG, new File(SharedResourcesPath, FILE_NAME).toString()));
         throw new RuntimeException(e);

ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml

@@ -29,18 +29,6 @@
       <reponame>HDP-UTILS</reponame>
     </repo>
   </os>
-  <os family="amazon2015">
-    <repo>
-      <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.6.0</baseurl>
-      <repoid>HDP-2.2</repoid>
-      <reponame>HDP</reponame>
-    </repo>
-    <repo>
-      <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl>
-      <repoid>HDP-UTILS-1.1.0.20</repoid>
-      <reponame>HDP-UTILS</reponame>
-    </repo>
-  </os>
   <os family="suse11">
     <repo>
       <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11sp3/2.x/updates/2.2.6.0</baseurl>

ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml

@@ -41,18 +41,6 @@
       <reponame>HDP-UTILS</reponame>
     </repo>
   </os>
-  <os family="amazon2015">
-    <repo>
-      <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0</baseurl>
-      <repoid>HDP-2.3</repoid>
-      <reponame>HDP</reponame>
-    </repo>
-    <repo>
-      <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl>
-      <repoid>HDP-UTILS-1.1.0.20</repoid>
-      <reponame>HDP-UTILS</reponame>
-    </repo>
-  </os>
   <os family="suse11">
     <repo>
       <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11sp3/2.x/updates/2.3.0.0</baseurl>

ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml

@@ -41,18 +41,6 @@
       <reponame>HDP-UTILS</reponame>
     </repo>
   </os>
-  <os family="redhat2015">
-    <repo>
-      <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/updates/2.4.0.0</baseurl>
-      <repoid>HDP-2.4</repoid>
-      <reponame>HDP</reponame>
-    </repo>
-    <repo>
-      <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl>
-      <repoid>HDP-UTILS-1.1.0.20</repoid>
-      <reponame>HDP-UTILS</reponame>
-    </repo>
-  </os>
   <os family="suse11">
     <repo>
       <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/suse11sp3/2.x/updates/2.4.0.0</baseurl>

ambari-server/src/test/python/TestOSCheck.py

@@ -30,7 +30,7 @@ from mock.mock import MagicMock
 
 from only_for_platform import os_distro_value, os_distro_value_linux
 
-from ambari_commons import OSCheck
+from ambari_commons import OSCheck, OSConst
 import os_check_type
 
 utils = __import__('ambari_server.utils').utils
@@ -50,7 +50,7 @@ class TestOSCheck(TestCase):
 
     # 1 - Any system
     mock_is_oracle_linux.return_value = False
-    mock_linux_distribution.return_value = ('my_os', '', '')
+    mock_linux_distribution.return_value = ('my_os', '2015.09', '')
     result = OSCheck.get_os_type()
     self.assertEquals(result, 'my_os')
 
@@ -66,13 +66,13 @@ class TestOSCheck(TestCase):
 
     # 3 - path exist: '/etc/oracle-release'
     mock_is_oracle_linux.return_value = True
-    mock_linux_distribution.return_value = ('some_os', '', '')
+    mock_linux_distribution.return_value = ('some_os', '1234', '')
     result = OSCheck.get_os_type()
     self.assertEquals(result, 'oraclelinux')
 
     # 4 - Common system
     mock_is_oracle_linux.return_value = False
-    mock_linux_distribution.return_value = ('CenToS', '', '')
+    mock_linux_distribution.return_value = ('CenToS', '4.56', '')
     result = OSCheck.get_os_type()
     self.assertEquals(result, 'centos')
 
@@ -99,31 +99,31 @@ class TestOSCheck(TestCase):
 
     # 1 - Any system
     mock_exists.return_value = False
-    mock_linux_distribution.return_value = ('MY_os', '', '')
+    mock_linux_distribution.return_value = ('MY_os', '5.6.7', '')
     result = OSCheck.get_os_family()
     self.assertEquals(result, 'my_os')
 
     # 2 - Redhat
     mock_exists.return_value = False
-    mock_linux_distribution.return_value = ('Centos Linux', '', '')
+    mock_linux_distribution.return_value = ('Centos Linux', '2.4', '')
     result = OSCheck.get_os_family()
     self.assertEquals(result, 'redhat')
 
     # 3 - Ubuntu
     mock_exists.return_value = False
-    mock_linux_distribution.return_value = ('Ubuntu', '', '')
+    mock_linux_distribution.return_value = ('Ubuntu', '14.04', '')
     result = OSCheck.get_os_family()
     self.assertEquals(result, 'ubuntu')
 
     # 4 - Suse
     mock_exists.return_value = False
     mock_linux_distribution.return_value = (
-    'suse linux enterprise server', '', '')
+    'suse linux enterprise server', '11.3', '')
     result = OSCheck.get_os_family()
     self.assertEquals(result, 'suse')
 
     mock_exists.return_value = False
-    mock_linux_distribution.return_value = ('SLED', '', '')
+    mock_linux_distribution.return_value = ('SLED', '1.2.3.4.5', '')
     result = OSCheck.get_os_family()
     self.assertEquals(result, 'suse')
 
@@ -141,7 +141,7 @@ class TestOSCheck(TestCase):
   def test_get_os_version(self, mock_linux_distribution):
 
     # 1 - Any system
-    mock_linux_distribution.return_value = ('', '123.45', '')
+    mock_linux_distribution.return_value = ('some_os', '123.45', '')
     result = OSCheck.get_os_version()
     self.assertEquals(result, '123.45')
 
@@ -159,7 +159,7 @@ class TestOSCheck(TestCase):
   def test_get_os_major_version(self, mock_linux_distribution):
 
     # 1
-    mock_linux_distribution.return_value = ('', '123.45.67', '')
+    mock_linux_distribution.return_value = ('abcd_os', '123.45.67', '')
     result = OSCheck.get_os_major_version()
     self.assertEquals(result, '123')
 
@@ -167,6 +167,21 @@ class TestOSCheck(TestCase):
     mock_linux_distribution.return_value = ('Suse', '11', '')
     result = OSCheck.get_os_major_version()
     self.assertEquals(result, '11')
+    
+  @patch.object(OSCheck, "os_distribution")
+  def test_aliases(self, mock_linux_distribution):
+    OSConst.OS_TYPE_ALIASES['qwerty_os123'] = 'aliased_os5'
+    OSConst.OS_FAMILY_COLLECTION.append({          
+          'name': 'aliased_os_family',
+          'os_list': ["aliased_os"]
+    })
+    
+    mock_linux_distribution.return_value = ('qwerty_os', '123.45.67', '')
+    
+    self.assertEquals(OSCheck.get_os_type(), 'aliased_os')
+    self.assertEquals(OSCheck.get_os_major_version(), '5')
+    self.assertEquals(OSCheck.get_os_version(), '5.45.67')
+    self.assertEquals(OSCheck.get_os_family(), 'aliased_os_family')
 
   @patch.object(OSCheck, "os_distribution")
   def test_get_os_release_name(self, mock_linux_distribution):

ambari-server/src/test/resources/os_family.json

@@ -1,45 +1,50 @@
 {
-  "redhat": {
-    "distro": [
-      "redhat",
-      "fedora",
-      "centos",
-      "oraclelinux"
-    ],
-    "versions": [
-      5,
-      6
-    ]
+  "mapping": {
+    "redhat": {
+      "distro": [
+        "redhat",
+        "fedora",
+        "centos",
+        "oraclelinux"
+      ],
+      "versions": [
+        5,
+        6
+      ]
+    },
+    "ubuntu": {
+      "distro": [
+        "ubuntu",
+        "debian"
+      ],
+      "versions": [
+        12
+      ]
+    },
+    "suse": {
+      "distro": [
+        "sles",
+        "sled",
+        "opensuse",
+        "suse"
+      ],
+      "versions": [
+        11
+      ]
+    },
+    "winsrv": {
+      "distro": [
+        "win2008server",
+        "win2008serverr2",
+        "win2012server",
+        "win2012serverr2"
+      ],
+      "versions": [
+        6
+      ]
+    }
   },
-  "ubuntu": {
-    "distro": [
-      "ubuntu",
-      "debian"
-    ],
-    "versions": [
-      12
-    ]
-  },
-  "suse": {
-    "distro": [
-      "sles",
-      "sled",
-      "opensuse",
-      "suse"
-    ],
-    "versions": [
-      11
-    ]
-  },
-  "winsrv": {
-    "distro": [
-      "win2008server",
-      "win2008serverr2",
-      "win2012server",
-      "win2012serverr2"
-    ],
-    "versions": [
-      6
-    ]
+  "aliases": {
+    "amazon2015": "amazon6"
   }
-}
+}