AMBARI-4420. ORA-01795: maximum number of expressions in a list is 1000 for Oracle DB. (swagle)

ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessor.java

@@ -320,6 +320,20 @@ public interface DBAccessor {
   public void setNullable(String tableName, DBAccessor.DBColumnInfo columnInfo, boolean nullable)
       throws SQLException;
 
+  public static enum DbType {
+    ORACLE,
+    MYSQL,
+    POSTGRES,
+    DERBY,
+    UNKNOWN
+  }
+
+  /**
+   * Get type of database platform
+   * @return @DbType
+   */
+  public DbType getDbType();
+
   /**
    * Capture column type
    */

ambari-server/src/main/java/org/apache/ambari/server/orm/DBAccessorImpl.java

@@ -17,22 +17,7 @@
  */
 package org.apache.ambari.server.orm;
 
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.regex.Pattern;
-import java.util.regex.Matcher;
-
+import com.google.inject.Inject;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.orm.helpers.ScriptRunner;
 import org.apache.ambari.server.orm.helpers.dbms.DbmsHelper;
@@ -55,8 +40,20 @@ import org.eclipse.persistence.sessions.DatabaseLogin;
 import org.eclipse.persistence.sessions.DatabaseSession;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-
-import com.google.inject.Inject;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Pattern;
 
 public class DBAccessorImpl implements DBAccessor {
   private static final Logger LOG = LoggerFactory.getLogger(DBAccessorImpl.class);
@@ -67,6 +64,7 @@ public class DBAccessorImpl implements DBAccessor {
   private DatabaseMetaData databaseMetaData;
   private static final String dbURLPatternString = "jdbc:(.*?):.*";
   private Pattern dbURLPattern = Pattern.compile(dbURLPatternString, Pattern.CASE_INSENSITIVE);
+  private DbType dbType;
 
   @Inject
   public DBAccessorImpl(Configuration configuration) {
@@ -101,14 +99,19 @@ public class DBAccessorImpl implements DBAccessor {
 
   protected DbmsHelper loadHelper(DatabasePlatform databasePlatform) {
     if (databasePlatform instanceof OraclePlatform) {
+      dbType = DbType.ORACLE;
       return new OracleHelper(databasePlatform);
     }else if (databasePlatform instanceof MySQLPlatform) {
+      dbType = DbType.MYSQL;
       return new MySqlHelper(databasePlatform);
     }else if (databasePlatform instanceof PostgreSQLPlatform) {
+      dbType = DbType.POSTGRES;
       return new PostgresHelper(databasePlatform);
     }else if (databasePlatform instanceof DerbyPlatform) {
+      dbType = DbType.DERBY;
       return new DerbyHelper(databasePlatform);
     } else {
+      dbType = DbType.UNKNOWN;
       return new GenericDbmsHelper(databasePlatform);
     }
   }
@@ -189,32 +192,8 @@ public class DBAccessorImpl implements DBAccessor {
     return result;
   }
 
-  protected String getDbType() {
-    String dbUrl = configuration.getDatabaseUrl();
-
-    // dbUrl will have the following format
-    // jdbc:{0}://{1}:{2}/{3},  type, host, port, name
-    // Most importantly, type is one of: postgresql, oracle:thin, mysql
-
-    if (null != dbUrl && !dbUrl.equals("")) {
-      Matcher m = dbURLPattern.matcher(dbUrl.toLowerCase());
-
-      if (m.find() && m.groupCount() == 1) {
-        String type = m.group(1);
-
-        if (type.contains(Configuration.POSTGRES_DB_NAME)) {
-          return Configuration.POSTGRES_DB_NAME;
-        } else if (type.contains(Configuration.ORACLE_DB_NAME)) {
-          return Configuration.ORACLE_DB_NAME;
-        } else if (type.contains(Configuration.MYSQL_DB_NAME)) {
-          return Configuration.MYSQL_DB_NAME;
-        } else if (type.contains(Configuration.DERBY_DB_NAME)) {
-          return Configuration.DERBY_DB_NAME;
-        }
-      }
-    }
-
-    throw new RuntimeException("Unable to determine database type.");
+  public DbType getDbType() {
+    return dbType;
   }
 
   @Override

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java

@@ -18,8 +18,8 @@
 
 package org.apache.ambari.server.orm.dao;
 
+import com.google.inject.Inject;
 import com.google.inject.Singleton;
-
 import javax.persistence.EntityManager;
 import javax.persistence.NoResultException;
 import javax.persistence.Query;
@@ -29,9 +29,19 @@ import javax.persistence.criteria.CriteriaQuery;
 import javax.persistence.criteria.Root;
 import java.util.Collections;
 import java.util.List;
+import static org.apache.ambari.server.orm.DBAccessor.DbType;
+import org.apache.ambari.server.orm.DBAccessor;
 
 @Singleton
 public class DaoUtils {
+  @Inject
+  private DBAccessor dbAccessor;
+
+  public static final int ORACLE_LIST_LIMIT = 999;
+
+  public DbType getDbType() {
+    return dbAccessor.getDbType();
+  }
 
   public <T> List<T> selectAll(EntityManager entityManager, Class<T> entityClass) {
     CriteriaBuilder criteriaBuilder = entityManager.getCriteriaBuilder();

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java

@@ -18,6 +18,7 @@
 
 package org.apache.ambari.server.orm.dao;
 
+import com.google.common.collect.Lists;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
 import com.google.inject.Singleton;
@@ -27,9 +28,6 @@ import org.apache.ambari.server.orm.RequiresSession;
 import org.apache.ambari.server.orm.entities.HostEntity;
 import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
 import org.apache.ambari.server.orm.entities.StageEntity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import javax.persistence.EntityManager;
 import javax.persistence.TypedQuery;
 import java.util.ArrayList;
@@ -38,6 +36,8 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import static org.apache.ambari.server.orm.DBAccessor.DbType.ORACLE;
+import static org.apache.ambari.server.orm.dao.DaoUtils.ORACLE_LIST_LIMIT;
 
 @Singleton
 public class HostRoleCommandDAO {
@@ -57,10 +57,23 @@ public class HostRoleCommandDAO {
     if (taskIds == null || taskIds.isEmpty()) {
       return Collections.emptyList();
     }
+
     TypedQuery<HostRoleCommandEntity> query = entityManagerProvider.get().createQuery(
       "SELECT task FROM HostRoleCommandEntity task WHERE task.taskId IN ?1 " +
         "ORDER BY task.taskId",
       HostRoleCommandEntity.class);
+
+    if (daoUtils.getDbType().equals(ORACLE) && taskIds.size() > ORACLE_LIST_LIMIT) {
+      List<HostRoleCommandEntity> result = new ArrayList<HostRoleCommandEntity>();
+
+      List<List<Long>> lists = Lists.partition(new ArrayList<Long>(taskIds), ORACLE_LIST_LIMIT);
+      for (List<Long> list : lists) {
+        result.addAll(daoUtils.selectList(query, list));
+      }
+
+      return result;
+    }
+
     return daoUtils.selectList(query, taskIds);
   }
 
@@ -99,6 +112,17 @@ public class HostRoleCommandDAO {
             "WHERE task.requestId IN ?1 AND task.taskId IN ?2 " +
             "ORDER BY task.taskId", Long.class
     );
+
+    if (daoUtils.getDbType().equals(ORACLE) && taskIds.size() > ORACLE_LIST_LIMIT) {
+      List<Long> result = new ArrayList<Long>();
+
+      List<List<Long>> lists = Lists.partition(new ArrayList<Long>(taskIds), ORACLE_LIST_LIMIT);
+      for (List<Long> taskIdList : lists) {
+        result.addAll(daoUtils.selectList(query, requestIds, taskIdList));
+      }
+
+      return result;
+    }
     return daoUtils.selectList(query, requestIds, taskIds);
   }
 

ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionDBAccessorImpl.java

@@ -16,24 +16,28 @@
  * limitations under the License.
  */
 package org.apache.ambari.server.actionmanager;
- 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
 
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Singleton;
+import com.google.inject.persist.PersistService;
+import com.google.inject.persist.UnitOfWork;
+import com.google.inject.util.Modules;
+import junit.framework.Assert;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.Role;
 import org.apache.ambari.server.RoleCommand;
 import org.apache.ambari.server.agent.ActionQueue;
 import org.apache.ambari.server.agent.CommandReport;
 import org.apache.ambari.server.api.services.BaseRequest;
+import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.ExecuteActionRequest;
 import org.apache.ambari.server.controller.HostsMap;
 import org.apache.ambari.server.controller.internal.RequestResourceFilter;
+import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.orm.DBAccessorImpl;
 import org.apache.ambari.server.orm.GuiceJpaInitializer;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
 import org.apache.ambari.server.orm.dao.ExecutionCommandDAO;
@@ -47,14 +51,13 @@ import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-
-import com.google.inject.Guice;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.persist.PersistService;
-import com.google.inject.persist.UnitOfWork;
-
-import junit.framework.Assert;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.apache.ambari.server.orm.DBAccessor.DbType.ORACLE;
 
 public class TestActionDBAccessorImpl {
   private static final Logger log = LoggerFactory.getLogger(TestActionDBAccessorImpl.class);
@@ -77,7 +80,9 @@ public class TestActionDBAccessorImpl {
 
   @Before
   public void setup() throws AmbariException {
-    injector = Guice.createInjector(new InMemoryDefaultTestModule());
+    InMemoryDefaultTestModule defaultTestModule = new InMemoryDefaultTestModule();
+    injector  = Guice.createInjector(Modules.override(defaultTestModule)
+      .with(new TestActionDBAccessorModule()));
     injector.getInstance(GuiceJpaInitializer.class);
     injector.injectMembers(this);
     clusters.addHost(hostName);
@@ -404,6 +409,76 @@ public class TestActionDBAccessorImpl {
     }
   }
 
+  private static class TestActionDBAccessorModule extends AbstractModule {
+    @Override
+    protected void configure() {
+      bind(DBAccessor.class).to(TestDBAccessorImpl.class);
+    }
+  }
+
+  @Singleton
+  static class TestDBAccessorImpl extends DBAccessorImpl {
+    private DbType dbTypeOverride = null;
+
+    @Inject
+    public TestDBAccessorImpl(Configuration configuration) {
+      super(configuration);
+    }
+
+    @Override
+    public DbType getDbType() {
+      if (dbTypeOverride != null) {
+        return dbTypeOverride;
+      }
+
+      return super.getDbType();
+    }
+
+    public void setDbTypeOverride(DbType dbTypeOverride) {
+      this.dbTypeOverride = dbTypeOverride;
+    }
+  }
+
+  @Test
+  public void testGet1000TasksFromOracleDB() throws Exception {
+    Stage s = new Stage(requestId, "/a/b", "cluster1", 1L, "action db accessor test",
+      "clusterHostInfo", "commandParamsStage", "hostParamsStage");
+    s.setStageId(stageId);
+    for (int i = 1000; i < 2002; i++) {
+      String host = "host" + i;
+
+      clusters.addHost(host);
+      clusters.getHost(host).persist();
+
+      s.addHostRoleExecutionCommand("host" + i, Role.HBASE_MASTER,
+        RoleCommand.START, null, "cluster1", "HBASE");
+    }
+
+    List<Stage> stages = new ArrayList<Stage>();
+    stages.add(s);
+    Request request = new Request(stages, clusters);
+    db.persistActions(request);
+
+    List<HostRoleCommandEntity> entities =
+      hostRoleCommandDAO.findByRequest(request.getRequestId());
+
+    assertEquals(1002, entities.size());
+    List<Long> taskIds = new ArrayList<Long>();
+    for (HostRoleCommandEntity entity : entities) {
+      taskIds.add(entity.getTaskId());
+    }
+
+    TestDBAccessorImpl testDBAccessorImpl =
+      (TestDBAccessorImpl) injector.getInstance(DBAccessor.class);
+
+    testDBAccessorImpl.setDbTypeOverride(ORACLE);
+
+    assertEquals(ORACLE, injector.getInstance(DBAccessor.class).getDbType());
+    entities = hostRoleCommandDAO.findByPKs(taskIds);
+    assertEquals("Tasks returned from DB match the ones created",
+      taskIds.size(), entities.size());
+  }
+
   private void populateActionDB(ActionDBAccessor db, String hostname,
       long requestId, long stageId) throws AmbariException {
     Stage s = createStubStage(hostname, requestId, stageId);

ambari-server/src/test/java/org/apache/ambari/server/agent/AgentResourceTest.java

@@ -18,28 +18,46 @@
 
 package org.apache.ambari.server.agent;
 
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import javax.ws.rs.core.MediaType;
-
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
 import com.google.inject.assistedinject.FactoryModuleBuilder;
 import com.google.inject.persist.jpa.AmbariJpaPersistModule;
-
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.spi.container.servlet.ServletContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
 import junit.framework.Assert;
-
 import org.apache.ambari.server.actionmanager.ActionDBAccessor;
 import org.apache.ambari.server.actionmanager.ActionManager;
 import org.apache.ambari.server.actionmanager.HostRoleCommandFactory;
 import org.apache.ambari.server.actionmanager.StageFactory;
 import org.apache.ambari.server.agent.rest.AgentResource;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.orm.DBAccessor;
 import org.apache.ambari.server.security.SecurityHelper;
 import org.apache.ambari.server.security.SecurityHelperImpl;
-import org.apache.ambari.server.state.*;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigFactory;
+import org.apache.ambari.server.state.ConfigImpl;
+import org.apache.ambari.server.state.Host;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.ambari.server.state.ServiceComponentFactory;
+import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.ServiceComponentHostFactory;
+import org.apache.ambari.server.state.ServiceComponentImpl;
+import org.apache.ambari.server.state.ServiceFactory;
+import org.apache.ambari.server.state.ServiceImpl;
 import org.apache.ambari.server.state.cluster.ClusterFactory;
 import org.apache.ambari.server.state.cluster.ClusterImpl;
 import org.apache.ambari.server.state.cluster.ClustersImpl;
@@ -57,19 +75,11 @@ import org.apache.commons.logging.LogFactory;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.junit.Test;
+import javax.ws.rs.core.MediaType;
 
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.spi.container.servlet.ServletContainer;
-import com.sun.jersey.test.framework.JerseyTest;
-import com.sun.jersey.test.framework.WebAppDescriptor;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class AgentResourceTest extends JerseyTest {
   static String PACKAGE_NAME = "org.apache.ambari.server.agent.rest";
@@ -280,6 +290,7 @@ public class AgentResourceTest extends JerseyTest {
       bind(AgentCommand.class).to(ExecutionCommand.class);
       bind(HeartBeatHandler.class).toInstance(handler);
       bind(AmbariMetaInfo.class).toInstance(ambariMetaInfo);
+      bind(DBAccessor.class).toInstance(mock(DBAccessor.class));
     }
 
     private void installDependencies() {

ambari-server/src/test/java/org/apache/ambari/server/configuration/ConfigurationTest.java

@@ -47,6 +47,7 @@ import org.junit.rules.TemporaryFolder;
 import org.junit.runner.RunWith;
 import org.powermock.api.easymock.PowerMock;
 import org.powermock.api.support.membermodification.MemberMatcher;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
@@ -54,9 +55,9 @@ import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 
-
 @RunWith(PowerMockRunner.class)
 @PrepareForTest({ Configuration.class })
+@PowerMockIgnore( {"javax.management.*"})
 public class ConfigurationTest {
   public TemporaryFolder temp = new TemporaryFolder();
   private Injector injector;

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AlertResourceProviderTest.java

@@ -17,6 +17,8 @@
  */
 package org.apache.ambari.server.controller.internal;
 
+import static org.apache.ambari.server.configuration.Configuration.JDBC_IN_MEMORY_URL;
+import static org.apache.ambari.server.configuration.Configuration.JDBC_IN_MEMROY_DRIVER;
 import static org.easymock.EasyMock.capture;
 import static org.easymock.EasyMock.captureLong;
 import static org.easymock.EasyMock.createMock;
@@ -31,12 +33,15 @@ import java.util.Set;
 
 import javax.persistence.EntityManager;
 
+import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.spi.Predicate;
 import org.apache.ambari.server.controller.spi.Request;
 import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.controller.utilities.PredicateBuilder;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.orm.DBAccessorImpl;
 import org.apache.ambari.server.orm.dao.AlertsDAO;
 import org.apache.ambari.server.orm.entities.AlertCurrentEntity;
 import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
@@ -80,10 +85,12 @@ public class AlertResourceProviderTest {
 
     Cluster cluster = EasyMock.createMock(Cluster.class);
     Clusters clusters = m_injector.getInstance(Clusters.class);
+    //Configuration config = m_injector.getInstance(Configuration.class);
+
     expect(m_amc.getClusters()).andReturn(clusters).atLeastOnce();
     expect(clusters.getCluster(capture(new Capture<String>()))).andReturn(cluster).atLeastOnce();
     expect(cluster.getClusterId()).andReturn(Long.valueOf(1L));
-    
+
     replay(m_amc, clusters, cluster);
     
     m_dao = m_injector.getInstance(AlertsDAO.class);
@@ -231,11 +238,19 @@ public class AlertResourceProviderTest {
       binder.bind(EntityManager.class).toInstance(EasyMock.createMock(EntityManager.class));
       binder.bind(AlertsDAO.class).toInstance(EasyMock.createMock(AlertsDAO.class));
       binder.bind(AmbariManagementController.class).toInstance(createMock(AmbariManagementController.class));
+      binder.bind(DBAccessor.class).to(DBAccessorImpl.class);
       
       Clusters clusters = EasyMock.createNiceMock(Clusters.class);
+      Configuration configuration = EasyMock.createMock(Configuration.class);
       
       binder.bind(Clusters.class).toInstance(clusters);
-      
+      binder.bind(Configuration.class).toInstance(configuration);
+
+      expect(configuration.getDatabaseUrl()).andReturn(JDBC_IN_MEMORY_URL).anyTimes();
+      expect(configuration.getDatabaseDriver()).andReturn(JDBC_IN_MEMROY_DRIVER).anyTimes();
+      expect(configuration.getDatabaseUser()).andReturn("test").anyTimes();
+      expect(configuration.getDatabasePassword()).andReturn("test").anyTimes();
+      replay(configuration);
     }
   }
 }

ambari-server/src/test/java/org/apache/ambari/server/orm/DBAccessorImplTest.java

@@ -77,6 +77,12 @@ public class DBAccessorImplTest {
     dbAccessor.createTable(tableName, columns, "id");
   }
 
+  @Test
+  public void testDbType() throws Exception {
+    DBAccessorImpl dbAccessor = injector.getInstance(DBAccessorImpl.class);
+    assertEquals(DBAccessor.DbType.DERBY, dbAccessor.getDbType());
+  }
+
   @Test
   public void testCreateTable() throws Exception {
     String tableName = getFreeTableName();