AMBARI-24992: Ambari Upgrade from 2.6.2.2 to 2.7.1.0 fails in Schema upgrade phase due to long certificate (#3480)

ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessor.java

@@ -663,6 +663,16 @@ public interface DBAccessor {
    */
   void changeColumnType(String tableName, String columnName, Class fromType, Class toType) throws SQLException;
 
+
+  /**
+   * Obtain column metadata information by given table and column name
+   * @param tableName name of the table
+   * @param columnName name of the column
+   * @return column information
+   * @throws SQLException
+   */
+  DBColumnInfo getColumnInfo(String tableName, String columnName) throws SQLException;
+
   /**
    * Queries the database to determine the name of the primary key constraint on
    * the specified table. Currently, this is only implemented for

ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java

@@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.math.BigDecimal;
 import java.nio.charset.Charset;
 import java.sql.Blob;
 import java.sql.Connection;
@@ -149,6 +150,49 @@ public class DBAccessorImpl implements DBAccessor {
     }
   }
 
+  /**
+   * Map SQL datatype to Java Class type
+   *
+   * @param type  SQL datatype
+   * @return Java class or null if no mapping found
+   */
+  private static Class<?> fromSqlTypeToClass(int type) {
+    switch (type) {
+      case Types.VARCHAR:
+      case Types.CHAR:
+      case Types.LONGVARCHAR:
+        return String.class;
+      case Types.NUMERIC:
+      case Types.DECIMAL:
+        return BigDecimal.class;
+      case Types.BIT:
+        return Boolean.class;
+      case Types.TINYINT:
+        return Byte.class;
+      case Types.SMALLINT:
+        return Short.class;
+      case Types.INTEGER:
+        return Integer.class;
+      case Types.BIGINT:
+        return Long.class;
+      case Types.FLOAT:
+      case Types.REAL:
+        return Float.class;
+      case Types.DOUBLE:
+        return Double.class;
+      case Types.BINARY:
+      case Types.VARBINARY:
+      case Types.LONGVARBINARY:
+        return Byte[].class;
+      case Types.DATE:
+        return java.sql.Date.class;
+      case Types.TIME:
+        return java.sql.Timestamp.class;
+      default:
+        return null;
+    }
+  }
+
   @Override
   public Connection getConnection() {
     return connection;
@@ -1231,6 +1275,39 @@ public class DBAccessorImpl implements DBAccessor {
     alterColumn(tableName, new DBColumnInfo(columnName, toType, null));
   }
 
+  /**
+   * Obtain column metadata information by given table and column name.
+   *
+   * Not able to return column default value.
+   *
+   * @param tableName  Name of the table
+   * @param columnName Name of the column
+   * @return Column information or null, if no column found
+   * @throws SQLException
+   */
+  @Override
+  public DBColumnInfo getColumnInfo(String tableName, String columnName) {
+    try {
+      String sqlQuery = String.format("SELECT %s FROM %s WHERE 1=2", columnName, convertObjectName(tableName));
+
+      try (Statement statement = getConnection().createStatement();
+           ResultSet rs = statement.executeQuery(sqlQuery)) {
+
+        ResultSetMetaData rsmd = rs.getMetaData();
+
+        return new DBColumnInfo(
+          rsmd.getColumnName(1),
+          fromSqlTypeToClass(rsmd.getColumnType(1)),
+          rsmd.getColumnDisplaySize(1),
+          null,
+          rsmd.isNullable(1) == ResultSetMetaData.columnNullable
+        );
+      }
+    } catch (SQLException e) {
+      return null;
+    }
+  }
+
   @Override
   public List<String> getIndexesList(String tableName, boolean unique)
     throws SQLException{

ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java

@@ -998,6 +998,11 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
     return false;
   }
 
+  /**
+   * Perform database schema transformation. Can work only before persist service start
+   * @throws AmbariException
+   * @throws SQLException
+   */
   protected abstract void executeDDLUpdates() throws AmbariException, SQLException;
 
   /**
@@ -1007,6 +1012,11 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
    */
   protected abstract void executePreDMLUpdates() throws AmbariException, SQLException;
 
+  /**
+   * Performs normal data upgrade
+   * @throws AmbariException
+   * @throws SQLException
+   */
   protected abstract void executeDMLUpdates() throws AmbariException, SQLException;
 
   @Override

ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java

@@ -181,8 +181,7 @@ public class SchemaUpgradeHelper {
     protected void configure() {
       super.configure();
       // Add binding to each newly created catalog
-      Multibinder<UpgradeCatalog> catalogBinder =
-        Multibinder.newSetBinder(binder(), UpgradeCatalog.class);
+      Multibinder<UpgradeCatalog> catalogBinder = Multibinder.newSetBinder(binder(), UpgradeCatalog.class);
       catalogBinder.addBinding().to(UpgradeCatalog251.class);
       catalogBinder.addBinding().to(UpgradeCatalog252.class);
       catalogBinder.addBinding().to(UpgradeCatalog260.class);
@@ -191,6 +190,7 @@ public class SchemaUpgradeHelper {
       catalogBinder.addBinding().to(UpgradeCatalog270.class);
       catalogBinder.addBinding().to(UpgradeCatalog271.class);
       catalogBinder.addBinding().to(UpgradeCatalog272.class);
+      catalogBinder.addBinding().to(UpgradeCatalog274.class);
       catalogBinder.addBinding().to(UpgradeCatalog275.class);
       catalogBinder.addBinding().to(UpgradeCatalog280.class);
       catalogBinder.addBinding().to(UpdateAlertScriptPaths.class);

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog274.java

@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.upgrade;
+
+import java.sql.SQLException;
+
+import javax.persistence.Table;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.orm.entities.AmbariConfigurationEntity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+
+/**
+ * The {@link UpgradeCatalog274} upgrades Ambari from 2.7.2 to 2.7.4.
+ */
+public class UpgradeCatalog274 extends AbstractUpgradeCatalog {
+
+  private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog274.class);
+  static final String AMBARI_CONFIGURATION_TABLE = AmbariConfigurationEntity.class.getAnnotation(Table.class).name();
+  static final String AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN = UpgradeCatalog270.AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN;
+  static final Integer AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN_LEN = 4000;
+
+
+  @Inject
+  public UpgradeCatalog274(Injector injector) {
+    super(injector);
+  }
+
+  @Override
+  public String getSourceVersion() {
+    return "2.7.2";
+  }
+
+  /**
+   * Perform database schema transformation. Can work only before persist service start
+   *
+   * @throws AmbariException
+   * @throws SQLException
+   */
+  @Override
+  protected void executeDDLUpdates() throws AmbariException, SQLException {
+    upgradeConfigurationTableValueMaxSize();
+  }
+
+  @Override
+  public String getTargetVersion() {
+    return "2.7.4";
+  }
+
+  /**
+   * Perform data insertion before running normal upgrade of data, requires started persist service
+   *
+   * @throws AmbariException
+   * @throws SQLException
+   */
+  @Override
+  protected void executePreDMLUpdates() throws AmbariException, SQLException {
+    // no actions needed
+  }
+
+  /**
+   * Performs normal data upgrade
+   *
+   * @throws AmbariException
+   * @throws SQLException
+   */
+  @Override
+  protected void executeDMLUpdates() throws AmbariException, SQLException {
+    // no actions needed
+  }
+
+
+  private void upgradeConfigurationTableValueMaxSize() throws SQLException {
+    DBAccessor.DBColumnInfo propertyColumn = dbAccessor.getColumnInfo(AMBARI_CONFIGURATION_TABLE,
+      AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN);
+
+    if (propertyColumn != null && propertyColumn.getType() != null &&
+      propertyColumn.getLength() < AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN_LEN) {
+
+      LOG.info("Updating column max size to {} for {}.{}", AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN_LEN,
+        AMBARI_CONFIGURATION_TABLE, AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN);
+
+      propertyColumn.setLength(AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN_LEN);
+      dbAccessor.alterColumn(AMBARI_CONFIGURATION_TABLE, propertyColumn);
+    }
+  }
+}

ambari-server/src/test/java/org/apache/ambari/server/orm/DBAccessorImplTest.java

@@ -814,6 +814,20 @@ public class DBAccessorImplTest {
     assertFalse(column1.equals(notEqualsColumn1Nullable));
   }
 
+  @Test
+  public void testFromSqlTypeToClass() throws Exception {
+    String tableName = getFreeTableName();
+    String columnName = "col1";
+
+    createMyTable(tableName, columnName);
+
+    DBAccessorImpl dbAccessor = injector.getInstance(DBAccessorImpl.class);
+    DBColumnInfo columnInfo =  dbAccessor.getColumnInfo(tableName, columnName);
+
+    assertEquals(columnName.toUpperCase(), columnInfo.getName());
+    assertEquals(String.class, columnInfo.getType());
+  }
+
   @Test
   public void testBuildQuery() throws Exception {
     String tableName = getFreeTableName();

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog274Test.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.upgrade;
+
+import static org.apache.ambari.server.upgrade.UpgradeCatalog274.AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN;
+import static org.apache.ambari.server.upgrade.UpgradeCatalog274.AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN_LEN;
+import static org.apache.ambari.server.upgrade.UpgradeCatalog274.AMBARI_CONFIGURATION_TABLE;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.newCapture;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+import org.apache.ambari.server.orm.DBAccessor;
+import org.easymock.Capture;
+import org.easymock.CaptureType;
+import org.easymock.EasyMockSupport;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.inject.Injector;
+
+public class UpgradeCatalog274Test {
+
+  private Injector injector;
+  private DBAccessor dbAccessor;
+
+  @Before
+  public void init() {
+    final EasyMockSupport easyMockSupport = new EasyMockSupport();
+    injector = easyMockSupport.createNiceMock(Injector.class);
+    dbAccessor = easyMockSupport.createNiceMock(DBAccessor.class);
+  }
+
+  @Test
+  public void testExecuteDDLUpdates() throws Exception {
+    DBAccessor.DBColumnInfo dbColumnInfo = new DBAccessor.DBColumnInfo(AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN,
+      String.class, 2000);
+
+    final Capture<DBAccessor.DBColumnInfo> alterPropertyValueColumnCapture = newCapture(CaptureType.ALL);
+    dbAccessor.getColumnInfo(eq(AMBARI_CONFIGURATION_TABLE), eq(AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN));
+    expectLastCall().andReturn(dbColumnInfo).once();
+
+    dbAccessor.alterColumn(eq(AMBARI_CONFIGURATION_TABLE), capture(alterPropertyValueColumnCapture));
+    expectLastCall().once();
+
+    replay(dbAccessor, injector);
+
+    UpgradeCatalog274 upgradeCatalog274 = new UpgradeCatalog274(injector);
+    upgradeCatalog274.dbAccessor = dbAccessor;
+    upgradeCatalog274.executeDDLUpdates();
+
+    final DBAccessor.DBColumnInfo alterPropertyValueColumn = alterPropertyValueColumnCapture.getValue();
+    Assert.assertEquals(AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN, alterPropertyValueColumn.getName());
+    Assert.assertEquals(String.class, alterPropertyValueColumn.getType());
+    Assert.assertEquals(AMBARI_CONFIGURATION_PROPERTY_VALUE_COLUMN_LEN, alterPropertyValueColumn.getLength());
+
+    verify(dbAccessor);
+  }
+}