HDDS-1376. Datanode exits while executing client command when scmId is null (#724)

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java

@@ -166,7 +166,6 @@ public class DatanodeStateMachine implements Closeable {
   private void start() throws IOException {
     long now = 0;
 
-    container.start();
     reportManager.init();
     initCommandHandlerThread(conf);
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java

@@ -106,7 +106,8 @@ public class VersionEndpointTask implements
           volumeSet.writeUnlock();
         }
 
-        ozoneContainer.getDispatcher().setScmId(scmId);
+        // Start the container services after getting the version information
+        ozoneContainer.start(scmId);
 
         EndpointStateMachine.EndPointStates nextState =
             rpcEndPoint.getState().getNextState();

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java

@@ -180,12 +180,13 @@ public class OzoneContainer {
    *
    * @throws IOException
    */
-  public void start() throws IOException {
+  public void start(String scmId) throws IOException {
     LOG.info("Attempting to start container services.");
     startContainerScrub();
     writeChannel.start();
     readChannel.start();
     hddsDispatcher.init();
+    hddsDispatcher.setScmId(scmId);
   }
 
   /**

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java

@@ -258,6 +258,21 @@ public class TestDatanodeStateMachine {
 
       task.execute(executorService);
       newState = task.await(10, TimeUnit.SECONDS);
+
+      // Wait for GetVersion call (called by task.execute) to finish. After
+      // Earlier task.execute called into GetVersion. Wait for the execution
+      // to finish and the endPointState to move to REGISTER state.
+      GenericTestUtils.waitFor(() -> {
+        for (EndpointStateMachine endpoint :
+            stateMachine.getConnectionManager().getValues()) {
+          if (endpoint.getState() !=
+              EndpointStateMachine.EndPointStates.REGISTER) {
+            return false;
+          }
+        }
+        return true;
+      }, 1000, 50000);
+
       // If we are in running state, we should be in running.
       Assert.assertEquals(DatanodeStateMachine.DatanodeStates.RUNNING,
           newState);

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java

@@ -68,7 +68,7 @@ public class TestCloseContainerCommandHandler {
     final DatanodeDetails datanodeDetails = randomDatanodeDetails();
     final OzoneContainer ozoneContainer =
         getOzoneContainer(conf, datanodeDetails);
-    ozoneContainer.start();
+    ozoneContainer.start(UUID.randomUUID().toString());
     try {
       final Container container =
           createContainer(conf, datanodeDetails, ozoneContainer);
@@ -105,7 +105,7 @@ public class TestCloseContainerCommandHandler {
     final DatanodeDetails datanodeDetails = randomDatanodeDetails();
     final OzoneContainer ozoneContainer =
         getOzoneContainer(conf, datanodeDetails);
-    ozoneContainer.start();
+    ozoneContainer.start(UUID.randomUUID().toString());
     try {
       final Container container =
           createContainer(conf, datanodeDetails, ozoneContainer);
@@ -140,7 +140,7 @@ public class TestCloseContainerCommandHandler {
     final DatanodeDetails datanodeDetails = randomDatanodeDetails();
     final OzoneContainer ozoneContainer =
         getOzoneContainer(conf, datanodeDetails);
-    ozoneContainer.start();
+    ozoneContainer.start(UUID.randomUUID().toString());
     try {
       final Container container =
           createContainer(conf, datanodeDetails, ozoneContainer);
@@ -188,7 +188,7 @@ public class TestCloseContainerCommandHandler {
     final DatanodeDetails datanodeDetails = randomDatanodeDetails();
     final OzoneContainer ozoneContainer =
         getOzoneContainer(conf, datanodeDetails);
-    ozoneContainer.start();
+    ozoneContainer.start(UUID.randomUUID().toString());
     try {
       final Container container =
           createContainer(conf, datanodeDetails, ozoneContainer);
@@ -224,7 +224,7 @@ public class TestCloseContainerCommandHandler {
     final DatanodeDetails datanodeDetails = randomDatanodeDetails();
     final OzoneContainer ozoneContainer = getOzoneContainer(
         conf, datanodeDetails);
-    ozoneContainer.start();
+    ozoneContainer.start(UUID.randomUUID().toString());
     try {
       final Container container = createContainer(
           conf, datanodeDetails, ozoneContainer);
@@ -277,7 +277,6 @@ public class TestCloseContainerCommandHandler {
     Mockito.when(context.getParent()).thenReturn(datanodeStateMachine);
     final OzoneContainer ozoneContainer = new  OzoneContainer(
         datanodeDetails, conf, context, null);
-    ozoneContainer.getDispatcher().setScmId(UUID.randomUUID().toString());
     return ozoneContainer;
   }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java

@@ -82,9 +82,8 @@ public class TestOzoneContainer {
       Mockito.when(dsm.getDatanodeDetails()).thenReturn(datanodeDetails);
       Mockito.when(context.getParent()).thenReturn(dsm);
       container = new OzoneContainer(datanodeDetails, conf, context, null);
-      //Setting scmId, as we start manually ozone container.
-      container.getDispatcher().setScmId(UUID.randomUUID().toString());
-      container.start();
+      //Set scmId and manually start ozone container.
+      container.start(UUID.randomUUID().toString());
 
       XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
       client.connect();

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java

@@ -155,9 +155,8 @@ public class TestOzoneContainerWithTLS {
           OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
 
       container = new OzoneContainer(dn, conf, getContext(dn), null);
-      //Setting scmId, as we start manually ozone container.
-      container.getDispatcher().setScmId(UUID.randomUUID().toString());
-      container.start();
+      //Set scmId and manually start ozone container.
+      container.start(UUID.randomUUID().toString());
 
       XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
       client.connect();

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java

@@ -143,9 +143,8 @@ public class TestSecureOzoneContainer {
 
       DatanodeDetails dn = TestUtils.randomDatanodeDetails();
       container = new OzoneContainer(dn, conf, getContext(dn), caClient);
-      //Setting scmId, as we start manually ozone container.
-      container.getDispatcher().setScmId(UUID.randomUUID().toString());
-      container.start();
+      //Set scmId and manually start ozone container.
+      container.start(UUID.randomUUID().toString());
 
       UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
           "user1",  new String[] {"usergroup"});