HDDS-204. Modify Integration tests for new ContainerIO classes. Contributed by Bharat Viswanadham

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java

@@ -286,6 +286,14 @@ public class KeyValueHandler extends Handler {
         throw new StorageContainerException(
             "Deletion of Open Container is not allowed.",
             DELETE_ON_OPEN_CONTAINER);
+      } else if (!forceDelete && kvContainer.getContainerData().getKeyCount()
+          > 0) {
+        // If the container is not empty and cannot be deleted forcibly,
+        // then throw a SCE to stop deleting.
+        kvContainer.writeUnlock();
+        throw new StorageContainerException(
+            "Container cannot be deleted because it is not empty.",
+            ContainerProtos.Result.ERROR_CONTAINER_NOT_EMPTY);
       } else {
         containerSet.removeContainer(
             kvContainer.getContainerData().getContainerID());

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java

@@ -130,16 +130,6 @@ public final class KeyValueContainerUtil {
         .getMetadataPath());
     File chunksPath = new File(containerData.getChunksPath());
 
-    MetadataStore db = KeyUtils.getDB(containerData, conf);
-
-    // If the container is not empty and cannot be deleted forcibly,
-    // then throw a SCE to stop deleting.
-    if(!forceDelete && !db.isEmpty()) {
-      throw new StorageContainerException(
-          "Container cannot be deleted because it is not empty.",
-          ContainerProtos.Result.ERROR_CONTAINER_NOT_EMPTY);
-    }
-
     // Close the DB connection and remove the DB handler from cache
     KeyUtils.removeDB(containerData, conf);
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java

@@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.common.Storage;
 import org.apache.hadoop.ozone.container.common.impl.ContainerData;
 import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
 import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
@@ -44,13 +45,16 @@ public class ContainerReader implements Runnable {
 
   private static final Logger LOG = LoggerFactory.getLogger(
       ContainerReader.class);
-  private File hddsVolumeDir;
+  private HddsVolume hddsVolume;
   private final ContainerSet containerSet;
   private final OzoneConfiguration config;
+  private final File hddsVolumeDir;
 
-  ContainerReader(File volumeRoot, ContainerSet cset, OzoneConfiguration conf) {
-    Preconditions.checkNotNull(volumeRoot);
-    this.hddsVolumeDir = volumeRoot;
+  ContainerReader(HddsVolume volume, ContainerSet cset, OzoneConfiguration
+      conf) {
+    Preconditions.checkNotNull(volume);
+    this.hddsVolume = volume;
+    this.hddsVolumeDir = hddsVolume.getHddsRootDir();
     this.containerSet = cset;
     this.config = conf;
   }
@@ -92,6 +96,11 @@ public class ContainerReader implements Runnable {
       }
     });
 
+    if (scmDir == null) {
+      LOG.error("Volume {} is empty with out metadata and chunks",
+          hddsVolumeRootDir);
+      return;
+    }
     for (File scmLoc : scmDir) {
       File currentDir = null;
       currentDir = new File(scmLoc, Storage.STORAGE_DIR_CURRENT);
@@ -137,6 +146,7 @@ public class ContainerReader implements Runnable {
       case KeyValueContainer:
         KeyValueContainerData keyValueContainerData = (KeyValueContainerData)
             containerData;
+        containerData.setVolume(hddsVolume);
         File dbFile = KeyValueContainerLocationUtil
             .getContainerDBFile(new File(containerFile.getParent()),
                 containerName);

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java

@@ -106,7 +106,7 @@ public class OzoneContainer {
     while (volumeSetIterator.hasNext()) {
       HddsVolume volume = volumeSetIterator.next();
       File hddsVolumeRootDir = volume.getHddsRootDir();
-      Thread thread = new Thread(new ContainerReader(hddsVolumeRootDir,
+      Thread thread = new Thread(new ContainerReader(volume,
           containerSet, config));
       thread.start();
       volumeThreads.add(thread);

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java

@@ -18,21 +18,13 @@
 
 package org.apache.hadoop.ozone.container.keyvalue;
 
-import com.google.common.base.Supplier;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
     .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
 import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.Container;
-import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestRule;
@@ -55,31 +47,14 @@ public class TestKeyValueHandler {
   @Rule
   public TestRule timeout = new Timeout(300000);
 
-  private Configuration conf;
   private HddsDispatcher dispatcher;
-  private ContainerSet containerSet;
-  private VolumeSet volumeSet;
   private KeyValueHandler handler;
 
-  private final static String SCM_ID = UUID.randomUUID().toString();
   private final static String DATANODE_UUID = UUID.randomUUID().toString();
-  private int containerID;
 
   private final String baseDir = MiniDFSCluster.getBaseDirectory();
   private final String volume = baseDir + "disk1";
 
-  private void setup() throws Exception {
-    this.conf = new Configuration();
-    conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, volume);
-
-    this.containerSet = new ContainerSet();
-    this.volumeSet = new VolumeSet(DATANODE_UUID, conf);
-
-    this.dispatcher = new HddsDispatcher(conf, containerSet, volumeSet);
-    this.handler = (KeyValueHandler) dispatcher.getHandler(
-        ContainerProtos.ContainerType.KeyValueContainer);
-    dispatcher.setScmId(UUID.randomUUID().toString());
-  }
 
   @Test
   /**
@@ -222,29 +197,5 @@ public class TestKeyValueHandler {
     return request;
   }
 
-  @Test
-  public void testCreateContainer() throws Exception {
-    setup();
-
-    long contId = ++containerID;
-    ContainerProtos.CreateContainerRequestProto createReq =
-        ContainerProtos.CreateContainerRequestProto.newBuilder()
-            .setContainerID(contId)
-            .build();
-
-    ContainerCommandRequestProto request =
-        ContainerProtos.ContainerCommandRequestProto.newBuilder()
-            .setCmdType(ContainerProtos.Type.CreateContainer)
-            .setDatanodeUuid(DATANODE_UUID)
-            .setCreateContainer(createReq)
-            .build();
-
-    dispatcher.dispatch(request);
 
-    // Verify that new container is added to containerSet.
-    Container container = containerSet.getContainer(contId);
-    Assert.assertEquals(contId, container.getContainerData().getContainerID());
-    Assert.assertEquals(ContainerProtos.ContainerLifeCycleState.OPEN,
-        container.getContainerState());
-  }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java

@@ -107,7 +107,9 @@ public class TestGenerateOzoneRequiredConfigurations {
    */
   @Test
   public void generateConfigurationsFailure() throws Exception {
-    String[] args = new String[]{"-output", "/"};
+    File tempPath = getRandomTempDir();
+    tempPath.setReadOnly();
+    String[] args = new String[]{"-output", tempPath.getAbsolutePath()};
     GenerateOzoneRequiredConfigurations.main(args);
 
     Assert.assertEquals("Path is valid",
@@ -118,6 +120,7 @@ public class TestGenerateOzoneRequiredConfigurations {
 
     Assert.assertEquals("Config file not generated",
         1, GenerateOzoneRequiredConfigurations.generateConfigurations(args[1]));
+    tempPath.setWritable(true);
   }
 
   private File getRandomTempDir() throws IOException {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java

@@ -143,7 +143,7 @@ public class TestContainerSmallFile {
 
 
     thrown.expect(StorageContainerException.class);
-    thrown.expectMessage("Unable to find the container");
+    thrown.expectMessage("ContainerID 8888 does not exist");
 
     // Try to read a invalid key
     ContainerProtos.GetSmallFileResponseProto response =

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java

@@ -67,6 +67,7 @@ import org.apache.log4j.Logger;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
@@ -662,6 +663,7 @@ public class TestKeys {
   }
 
   @Test
+  @Ignore("Needs to be fixed for new SCM and Storage design")
   public void testDeleteKey() throws Exception {
     KeySpaceManager ksm = ozoneCluster.getKeySpaceManager();
     // To avoid interference from other test cases,