AMBARI-1676. Ambari upgrade to 1.3.0 (core support). (smohanty)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/trunk@1461385 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -12,6 +12,8 @@ Trunk (unreleased changes):
 
  NEW FEATURES
 
+ AMBARI-1676. Ambari upgrade to 1.3.0 (core support). (smohanty)
+
  AMBARI-1708. Remove all hardcoded ports from agent scripts to read from 
  configs. (swagle)
 

ambari-server/pom.xml

@@ -253,13 +253,13 @@
               <groupname>root</groupname>
               <sources>
                 <source>
-                  <location>src/main/resources/upgrade/ddl/Ambari-DDL-Postgres-UPGRADE-1.2.2.sql</location>
+                  <location>src/main/resources/upgrade/ddl/Ambari-DDL-Postgres-UPGRADE-1.3.0.sql</location>
                 </source>
                 <source>
-                  <location>src/main/resources/upgrade/ddl/Ambari-DDL-Postgres-UPGRADE-1.2.2.Fix.sql</location>
+                  <location>src/main/resources/upgrade/ddl/Ambari-DDL-Postgres-UPGRADE-1.3.0.Fix.sql</location>
                 </source>
                 <source>
-                  <location>src/main/resources/upgrade/ddl/Ambari-DDL-Postgres-UPGRADE-1.2.2.Check.sql</location>
+                  <location>src/main/resources/upgrade/ddl/Ambari-DDL-Postgres-UPGRADE-1.3.0.Check.sql</location>
                 </source>
               </sources>
             </mapping>

ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java

@@ -407,7 +407,8 @@ public class HeartBeatHandler {
 
     hostObject.handleEvent(new HostRegistrationRequestEvent(hostname,
         null != register.getPublicHostname() ? register.getPublicHostname() : hostname,
-        new AgentVersion("v1"), now, register.getHardwareProfile(), register.getAgentEnv()));
+        new AgentVersion(register.getAgentVersion()), now, register.getHardwareProfile(),
+        register.getAgentEnv()));
     RegistrationResponse response = new RegistrationResponse();
     if (cmds.isEmpty()) {
       //No status commands needed let the fsm know that status step is done

ambari-server/src/main/java/org/apache/ambari/server/agent/Register.java

@@ -32,6 +32,7 @@ public class Register {
   private HostInfo hardwareProfile;
   private String publicHostname;
   private AgentEnv agentEnv;
+  private String agentVersion;
 
   @JsonProperty("responseId")
   public int getResponseId() {
@@ -72,7 +73,7 @@ public class Register {
   }
   
   public void setPublicHostname(String name) {
-    publicHostname = name;
+    this.publicHostname = name;
   }
   
   public AgentEnv getAgentEnv() {
@@ -80,7 +81,15 @@ public class Register {
   }
   
   public void setAgentEnv(AgentEnv env) {
-    agentEnv = env;
+    this.agentEnv = env;
+  }
+
+  public String getAgentVersion() {
+    return agentVersion;
+  }
+
+  public void setAgentVersion(String agentVersion) {
+    this.agentVersion = agentVersion;
   }
 
   @Override

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/MetainfoDAO.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.orm.dao;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.persist.Transactional;
+import org.apache.ambari.server.orm.entities.MetainfoEntity;
+
+import javax.persistence.EntityManager;
+import javax.persistence.TypedQuery;
+import java.util.Collection;
+
+public class MetainfoDAO {
+
+  @Inject
+  Provider<EntityManager> entityManagerProvider;
+  @Inject
+  DaoUtils daoUtils;
+
+  @Transactional
+  public MetainfoEntity findByKey(String key) {
+    return entityManagerProvider.get().find(MetainfoEntity.class, key);
+  }
+
+  @Transactional
+  public Collection<MetainfoEntity> findAll() {
+    TypedQuery<MetainfoEntity> query =
+        entityManagerProvider.get().createQuery("SELECT metainfo FROM MetainfoEntity metainfo", MetainfoEntity.class);
+    return daoUtils.selectList(query);
+  }
+
+  @Transactional
+  public void refresh(MetainfoEntity metainfoEntity) {
+    entityManagerProvider.get().refresh(metainfoEntity);
+  }
+
+  @Transactional
+  public void create(MetainfoEntity metainfoEntity) {
+    entityManagerProvider.get().persist(metainfoEntity);
+  }
+
+  @Transactional
+  public MetainfoEntity merge(MetainfoEntity metainfoEntity) {
+    return entityManagerProvider.get().merge(metainfoEntity);
+  }
+
+  @Transactional
+  public void remove(MetainfoEntity metainfoEntity) {
+    entityManagerProvider.get().remove(merge(metainfoEntity));
+  }
+
+  @Transactional
+  public void removeByHostName(String key) {
+    remove(findByKey(key));
+  }
+
+}

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/MetainfoEntity.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.orm.entities;
+
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Table(name = "metainfo", schema = "ambari", catalog = "")
+@Entity
+public class MetainfoEntity {
+
+  private String metainfoName;
+  private String metainfoValue;
+
+  @Column(name = "\"metainfo_key\"", length = 255)
+  @Id
+  public String getMetainfoName() {
+    return metainfoName;
+  }
+
+  public void setMetainfoName(String metainfoName) {
+    this.metainfoName = metainfoName;
+  }
+
+  @Column(name = "\"metainfo_value\"", length = 32000)
+  public String getMetainfoValue() {
+    return metainfoValue;
+  }
+
+  public void setMetainfoValue(String metainfoValue) {
+    this.metainfoValue = metainfoValue;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    MetainfoEntity that = (MetainfoEntity) o;
+
+    if (metainfoName != null ? !metainfoName.equals(that.metainfoName) : that.metainfoName != null) {
+      return false;
+    }
+    if (metainfoValue != null ? !metainfoValue.equals(that.metainfoValue) : that.metainfoValue != null) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    int result = metainfoName != null ? metainfoName.hashCode() : 0;
+    result = 31 * result + (metainfoValue != null ? metainfoValue.hashCode() : 0);
+    return result;
+  }
+}

ambari-server/src/main/java/org/apache/ambari/server/state/StackId.java

@@ -18,6 +18,8 @@
 
 package org.apache.ambari.server.state;
 
+import org.apache.ambari.server.utils.VersionUtils;
+
 public class StackId implements Comparable<StackId> {
 
   private static final String NAME_SEPARATOR = "-";
@@ -105,21 +107,7 @@ public class StackId implements Comparable<StackId> {
 
     int returnValue = getStackName().compareTo(other.getStackName());
     if (returnValue == 0) {
-      String[] thisVersionParts = getStackVersion().split("\\.");
-      String[] thatVersionParts = other.getStackVersion().split("\\.");
-      int length = Math.max(thisVersionParts.length, thatVersionParts.length);
-      for (int i = 0; i < length; i++) {
-        int stack1Part = i < thisVersionParts.length ?
-            Integer.parseInt(thisVersionParts[i]) : 0;
-        int stack2Part = i < thatVersionParts.length ?
-            Integer.parseInt(thatVersionParts[i]) : 0;
-        if (stack1Part < stack2Part) {
-          return -1;
-        }
-        if (stack1Part > stack2Part) {
-          return 1;
-        }
-      }
+      returnValue = VersionUtils.compareVersions(getStackVersion(), other.getStackVersion());
     } else {
       throw new RuntimeException("StackId with different names cannot be compared.");
     }

ambari-server/src/main/java/org/apache/ambari/server/utils/VersionUtils.java

@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.utils;
+
+/**
+ * Provides various utility functions to be used for version handling.
+ * The compatibility matrix between server, agent, store can also be maintained
+ * in this class. Currently, exact match is required between all the three.
+ */
+public class VersionUtils {
+  /**
+   * Compares two versions strings of the form N.N.N.N
+   *
+   * @param version1
+   * @param version2
+   * @param maxLengthToCompare The maximum length to compare - 2 means only Major and Minor
+   *                           0 to compare the whole version strings
+   * @return 0 if both are equal up to the length compared, -1 if first one is lower, +1 otherwise
+   */
+  public static int compareVersions(String version1, String version2, int maxLengthToCompare) {
+    if (version1 == null || version1.isEmpty()) {
+      throw new IllegalArgumentException("version1 cannot be null or empty");
+    }
+    if (version2 == null || version2.isEmpty()) {
+      throw new IllegalArgumentException("version2 cannot be null or empty");
+    }
+    if (maxLengthToCompare < 0) {
+      throw new IllegalArgumentException("maxLengthToCompare cannot be less than 0");
+    }
+
+    String[] version1Parts = version1.split("\\.");
+    String[] version2Parts = version2.split("\\.");
+
+    int length = Math.max(version1Parts.length, version2Parts.length);
+    length = maxLengthToCompare == 0 || maxLengthToCompare > length ? length : maxLengthToCompare;
+    for (int i = 0; i < length; i++) {
+      int stack1Part = i < version1Parts.length ?
+          Integer.parseInt(version1Parts[i]) : 0;
+      int stack2Part = i < version2Parts.length ?
+          Integer.parseInt(version2Parts[i]) : 0;
+      if (stack1Part < stack2Part) {
+        return -1;
+      }
+      if (stack1Part > stack2Part) {
+        return 1;
+      }
+    }
+
+    return 0;
+  }
+
+  /**
+   * Compares two versions strings of the form N.N.N.N
+   *
+   * @param version1
+   * @param version2
+   * @return 0 if both are equal, -1 if first one is lower, +1 otherwise
+   */
+  public static int compareVersions(String version1, String version2) {
+    return compareVersions(version1, version2, 0);
+  }
+
+  /**
+   * Compares two version for equality
+   *
+   * @param version1
+   * @param version2
+   * @return true if versions are equal; false otherwise
+   */
+  public static boolean areVersionsEqual(String version1, String version2) {
+    return 0 == compareVersions(version1, version2, 0);
+  }
+
+  /**
+   * Checks if the two versions are compatible.
+   * TODO A relaxed check can be implemented where a server version is compatible to
+   * TODO more than one versions of agent and store.
+   *
+   * @param serverVersion The version of the server
+   * @param versionToCheck The version of the agent or the store
+   * @return true if the versions are compatible
+   */
+  public static boolean areVersionsCompatible(String serverVersion, String versionToCheck) {
+    return areVersionsEqual(serverVersion, versionToCheck);
+  }
+}

ambari-server/src/main/python/ambari-server.py

@@ -286,7 +286,7 @@ def check_db_consistency(args, file):
     # Assumes that the output is of the form ...\n<count>
     print_info_msg("Parsing output: " + outdata)
     lines = outdata.splitlines()
-    if (lines[-1] == '3'):
+    if (lines[-1] == '3' or lines[-1] == '0'):
       return 0
   return -1
 
@@ -1120,7 +1120,7 @@ def main():
                       help="File with drop script")
   parser.add_option('-u', '--upgrade-script-file', default="/var/lib/"
                               "ambari-server/resources/upgrade/ddl/"
-                              "Ambari-DDL-Postgres-UPGRADE-1.2.2.sql",
+                              "Ambari-DDL-Postgres-UPGRADE-1.3.0.sql",
                       help="File with upgrade script")
   parser.add_option('-t', '--upgrade-stack-script-file', default="/var/lib/"
                               "ambari-server/resources/upgrade/dml/"

ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql

@@ -113,9 +113,11 @@ CREATE TABLE ambari.key_value_store ("key" VARCHAR(255), "value" VARCHAR, PRIMAR
 GRANT ALL PRIVILEGES ON TABLE ambari.key_value_store TO :username;
 
 CREATE TABLE ambari.hostconfigmapping (cluster_id bigint NOT NULL, host_name VARCHAR(255) NOT NULL, type_name VARCHAR(255) NOT NULL, version_tag VARCHAR(255) NOT NULL, service_name VARCHAR(255), create_timestamp BIGINT NOT NULL, selected INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (cluster_id, host_name, type_name, create_timestamp));
-
 GRANT ALL PRIVILEGES ON TABLE ambari.hostconfigmapping TO :username;
 
+CREATE TABLE ambari.metainfo ("metadata_name" VARCHAR(255), "metadata_value" VARCHAR, PRIMARY KEY("metadata_name"));
+GRANT ALL PRIVILEGES ON TABLE ambari.metainfo TO :username;
+
 ALTER TABLE ambari.clusterconfig ADD CONSTRAINT FK_clusterconfig_cluster_id FOREIGN KEY (cluster_id) REFERENCES ambari.clusters (cluster_id);
 ALTER TABLE ambari.clusterservices ADD CONSTRAINT FK_clusterservices_cluster_id FOREIGN KEY (cluster_id) REFERENCES ambari.clusters (cluster_id);
 ALTER TABLE ambari.clusterconfigmapping ADD CONSTRAINT FK_clusterconfigmapping_cluster_id FOREIGN KEY (cluster_id) REFERENCES ambari.clusters (cluster_id);
@@ -170,6 +172,9 @@ select 'admin','538916f8943ec225d97a9a86a2c6ec0818c1cd400e09e03b660fdaaec4af29dd
 insert into ambari.user_roles(role_name, user_id)
 select 'admin',(select user_id from ambari.users where user_name='admin' and ldap_user=false);
 
+insert into ambari.metainfo(metadata_name, metadata_value)
+select 'version','1.3.0';
+
 COMMIT;
 
 -- ambari log4j DDL

ambari-server/src/main/resources/Ambari-DDL-Postgres-DROP.sql

@@ -66,6 +66,7 @@ DROP TABLE ambari.clusterconfig CASCADE;
 DROP TABLE ambari.clusterconfigmapping CASCADE;
 DROP TABLE ambari.user_roles CASCADE;
 DROP TABLE ambari.key_value_store CASCADE;
+DROP TABLE ambari.metainfo CASCADE;
 DROP SEQUENCE ambari.host_role_command_task_id_seq;
 DROP SEQUENCE ambari.clusters_cluster_id_seq;
 

ambari-server/src/main/resources/META-INF/persistence.xml

@@ -37,6 +37,7 @@
     <class>org.apache.ambari.server.orm.entities.KeyValueEntity</class>
     <class>org.apache.ambari.server.orm.entities.ClusterConfigMappingEntity</class>
     <class>org.apache.ambari.server.orm.entities.HostConfigMappingEntity</class>
+    <class>org.apache.ambari.server.orm.entities.MetainfoEntity</class>
 
     <properties>
       <property name="javax.persistence.jdbc.url" value="jdbc:postgresql://localhost/ambari" />
@@ -70,6 +71,7 @@
     <class>org.apache.ambari.server.orm.entities.KeyValueEntity</class>
     <class>org.apache.ambari.server.orm.entities.ClusterConfigMappingEntity</class>
     <class>org.apache.ambari.server.orm.entities.HostConfigMappingEntity</class>
+    <class>org.apache.ambari.server.orm.entities.MetainfoEntity</class>
 
     <properties>
       <property name="javax.persistence.jdbc.url" value="jdbc:derby:memory:myDB;create=true" />

ambari-server/src/main/resources/upgrade/ddl/Ambari-DDL-Postgres-UPGRADE-1.2.1.sql → ambari-server/src/main/resources/upgrade/ddl/Ambari-DDL-Postgres-UPGRADE-1.3.0.Check.sql

@@ -17,5 +17,4 @@
 --
 \connect ambari;
 
-ALTER TABLE ambari.hosts
-  ALTER COLUMN disks_info TYPE VARCHAR(10000);
+COPY (SELECT count(*) FROM ambari.serviceconfigmapping WHERE service_name = 'MAPREDUCE') TO STDOUT WITH CSV;

ambari-server/src/main/resources/upgrade/ddl/Ambari-DDL-Postgres-UPGRADE-1.3.0.Fix.sql

@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+\connect ambari;
+
+INSERT INTO ambari.serviceconfigmapping (cluster_id, service_name, config_type, config_tag, timestamp)
+    SELECT cluster_id, 'MAPREDUCE', type_name, version_tag, create_timestamp from ambari.clusterconfig 
+        WHERE type_name = 'global' ORDER BY create_timestamp DESC LIMIT 1;
+
+INSERT INTO ambari.serviceconfigmapping (cluster_id, service_name, config_type, config_tag, timestamp)
+    SELECT cluster_id, 'MAPREDUCE', type_name, version_tag, create_timestamp from ambari.clusterconfig
+         WHERE type_name = 'mapred-site' ORDER BY create_timestamp DESC LIMIT 1;

ambari-server/src/main/resources/upgrade/ddl/Ambari-DDL-Postgres-UPGRADE-1.3.0.sql

@@ -17,9 +17,17 @@
 --
 \connect ambari;
 
+-- Upgrade from 1.2.0
+ALTER TABLE ambari.hosts
+  ALTER COLUMN disks_info TYPE VARCHAR(10000);
+
+-- Upgrade to 1.3.0
 ALTER TABLE ambari.clusterstate
   ADD COLUMN current_stack_version VARCHAR(255) NOT NULL;
 
+CREATE TABLE ambari.metainfo ("metadata_name" VARCHAR(255), "metadata_value" VARCHAR, PRIMARY KEY("metadata_name"));
+GRANT ALL PRIVILEGES ON TABLE ambari.metainfo TO :username;
+
 CREATE TABLE ambari.clusterconfigmapping (cluster_id bigint NOT NULL, type_name VARCHAR(255) NOT NULL, version_tag VARCHAR(255) NOT NULL, create_timestamp BIGINT NOT NULL, selected INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (cluster_id, type_name, create_timestamp));
 GRANT ALL PRIVILEGES ON TABLE ambari.clusterconfigmapping TO :username;
 ALTER TABLE ambari.clusterconfigmapping ADD CONSTRAINT FK_clusterconfigmapping_cluster_id FOREIGN KEY (cluster_id) REFERENCES ambari.clusters (cluster_id);
@@ -32,4 +40,11 @@ ALTER TABLE ambari.hostconfigmapping ADD CONSTRAINT FK_hostconfigmapping_host_na
 ALTER SEQUENCE ambari.host_role_command_task_id_seq INCREMENT BY 50;
 SELECT nextval('ambari.host_role_command_task_id_seq');
 
-ALTER TABLE ambari.stage ADD COLUMN request_context VARCHAR(255);
+ALTER TABLE ambari.stage ADD COLUMN request_context VARCHAR(255);
+
+BEGIN;
+
+insert into ambari.metainfo(metadata_name, metadata_value)
+select 'version','1.3.0';
+
+COMMIT;