Revert "HDDS-1474. ozone.scm.datanode.id config should take path for a dir (#792)" (#808)

This reverts commit 18cca583ac747f5736777a5aed7e5c02bff1bc0a.

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -296,10 +296,11 @@ public final class ScmConfigKeys {
 
   public static final int OZONE_SCM_DEFAULT_PORT =
       OZONE_SCM_DATANODE_PORT_DEFAULT;
-  // The path where datanode ID is to be written to.
+  // File Name and path where datanode ID is to written to.
   // if this value is not set then container startup will fail.
-  public static final String OZONE_SCM_DATANODE_ID_DIR =
-      "ozone.scm.datanode.id.dir";
+  public static final String OZONE_SCM_DATANODE_ID = "ozone.scm.datanode.id";
+
+  public static final String OZONE_SCM_DATANODE_ID_PATH_DEFAULT = "datanode.id";
 
   public static final String OZONE_SCM_DB_CACHE_SIZE_MB =
       "ozone.scm.db.cache.size.mb";

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -195,11 +195,6 @@ public final class OzoneConsts {
   public static final int INVALID_PORT = -1;
 
 
-  /**
-   * Default SCM Datanode ID file name.
-   */
-  public static final String OZONE_SCM_DATANODE_ID_FILE_DEFAULT = "datanode.id";
-
   // The ServiceListJSONServlet context attribute where OzoneManager
   // instance gets stored.
   public static final String OM_CONTEXT_ATTRIBUTE = "ozone.om";

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -883,7 +883,7 @@
     </description>
   </property>
   <property>
-    <name>ozone.scm.datanode.id.dir</name>
+    <name>ozone.scm.datanode.id</name>
     <value/>
     <tag>OZONE, MANAGEMENT</tag>
     <description>The path that datanodes will use to store the datanode ID.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java

@@ -23,7 +23,6 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.server.ServerUtils;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -381,9 +380,8 @@ public final class HddsServerUtil {
    * @return the path of datanode id as string
    */
   public static String getDatanodeIdFilePath(Configuration conf) {
-    String dataNodeIDDirPath =
-        conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR);
-    if (dataNodeIDDirPath == null) {
+    String dataNodeIDPath = conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID);
+    if (dataNodeIDPath == null) {
       File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
       if (metaDirPath == null) {
         // this means meta data is not found, in theory should not happen at
@@ -391,10 +389,9 @@ public final class HddsServerUtil {
         throw new IllegalArgumentException("Unable to locate meta data" +
             "directory when getting datanode id path");
       }
-      dataNodeIDDirPath = metaDirPath.toString();
+      dataNodeIDPath = new File(metaDirPath,
+          ScmConfigKeys.OZONE_SCM_DATANODE_ID_PATH_DEFAULT).toString();
     }
-    // Use default datanode id file name for file path
-    return new File(dataNodeIDDirPath,
-        OzoneConsts.OZONE_SCM_DATANODE_ID_FILE_DEFAULT).toString();
+    return dataNodeIDPath;
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java

@@ -320,10 +320,9 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
       throws IOException {
     String idFilePath = HddsServerUtil.getDatanodeIdFilePath(conf);
     if (idFilePath == null || idFilePath.isEmpty()) {
-      LOG.error("A valid path is needed for config setting {}",
-          ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR);
-      throw new IllegalArgumentException(
-          ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR +
+      LOG.error("A valid file path is needed for config setting {}",
+          ScmConfigKeys.OZONE_SCM_DATANODE_ID);
+      throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_DATANODE_ID +
           " must be defined. See" +
           " https://wiki.apache.org/hadoop/Ozone#Configuration" +
           " for details on configuring Ozone.");
@@ -351,10 +350,9 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
       throws IOException {
     String idFilePath = HddsServerUtil.getDatanodeIdFilePath(conf);
     if (idFilePath == null || idFilePath.isEmpty()) {
-      LOG.error("A valid path is needed for config setting {}",
-          ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR);
-      throw new IllegalArgumentException(
-          ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR +
+      LOG.error("A valid file path is needed for config setting {}",
+          ScmConfigKeys.OZONE_SCM_DATANODE_ID);
+      throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_DATANODE_ID +
           " must be defined. See" +
           " https://wiki.apache.org/hadoop/Ozone#Configuration" +
           " for details on configuring Ozone.");

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java

@@ -119,8 +119,8 @@ public class InitDatanodeState implements DatanodeState,
   private void persistContainerDatanodeDetails() {
     String dataNodeIDPath = HddsServerUtil.getDatanodeIdFilePath(conf);
     if (Strings.isNullOrEmpty(dataNodeIDPath)) {
-      LOG.error("A valid path is needed for config setting {}",
-          ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR);
+      LOG.error("A valid file path is needed for config setting {}",
+          ScmConfigKeys.OZONE_SCM_DATANODE_ID);
       this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN);
       return;
     }

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java

@@ -25,7 +25,6 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
 import org.apache.hadoop.ozone.container.common.statemachine
     .DatanodeStateMachine;
@@ -50,6 +49,7 @@ import org.slf4j.LoggerFactory;
 import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -117,8 +117,9 @@ public class TestDatanodeStateMachine {
     }
     conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
         new File(testRoot, "scm").getAbsolutePath());
-    path = new File(testRoot, "datanodeID").getAbsolutePath();
-    conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR, path);
+    path = Paths.get(path.toString(),
+        TestDatanodeStateMachine.class.getSimpleName() + ".id").toString();
+    conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ID, path);
     executorService = HadoopExecutors.newCachedThreadPool(
         new ThreadFactoryBuilder().setDaemon(true)
             .setNameFormat("Test Data Node State Machine Thread - %d").build());
@@ -211,8 +212,7 @@ public class TestDatanodeStateMachine {
     // There is no mini cluster started in this test,
     // create a ID file so that state machine could load a fake datanode ID.
     File idPath = new File(
-        conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR),
-        OzoneConsts.OZONE_SCM_DATANODE_ID_FILE_DEFAULT);
+        conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID));
     idPath.delete();
     DatanodeDetails datanodeDetails = getNewDatanodeDetails();
     DatanodeDetails.Port port = DatanodeDetails.newPort(
@@ -334,8 +334,7 @@ public class TestDatanodeStateMachine {
   public void testDatanodeStateMachineWithIdWriteFail() throws Exception {
 
     File idPath = new File(
-        conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR),
-        OzoneConsts.OZONE_SCM_DATANODE_ID_FILE_DEFAULT);
+        conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID));
     idPath.delete();
     DatanodeDetails datanodeDetails = getNewDatanodeDetails();
     DatanodeDetails.Port port = DatanodeDetails.newPort(
@@ -397,10 +396,10 @@ public class TestDatanodeStateMachine {
     /** Port out of range **/
     confList.add(Maps.immutableEntry(
         ScmConfigKeys.OZONE_SCM_NAMES, "scm:123456"));
-    // Invalid ozone.scm.datanode.id.dir
+    // Invalid ozone.scm.datanode.id
     /** Empty **/
     confList.add(Maps.immutableEntry(
-        ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR, ""));
+        ScmConfigKeys.OZONE_SCM_DATANODE_ID, ""));
 
     confList.forEach((entry) -> {
       Configuration perTestConf = new Configuration(conf);

hadoop-hdds/docs/content/Settings.md

@@ -105,14 +105,14 @@ Here is an example,
       </property>
   {{< /highlight >}}
 
- * **ozone.scm.datanode.id.dir** Data nodes generate a Unique ID called Datanode
- ID. This identity is written to the file datanode.id in a directory specified by this path. *Data nodes
+ * **ozone.scm.datanode.id** Data nodes generate a Unique ID called Datanode
+ ID. This identity is written to the file specified by this path. *Data nodes
     will create this path if it doesn't exist already.*
 
 Here is an  example,
 {{< highlight xml >}}
    <property>
-      <name>ozone.scm.datanode.id.dir</name>
+      <name>ozone.scm.datanode.id</name>
       <value>/data/disk1/meta/node/datanode.id</value>
    </property>
 {{< /highlight >}}

hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config

@@ -18,7 +18,7 @@ OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=true
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.scm.client.address=scm

hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config

@@ -21,7 +21,7 @@ OZONE-SITE.XML_ozone.om.address.om3=om3
 OZONE-SITE.XML_ozone.om.ratis.enable=true
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed

hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config

@@ -18,7 +18,7 @@ OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon

hadoop-ozone/dist/src/main/compose/ozone/docker-config

@@ -18,7 +18,7 @@ OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed

hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config

@@ -18,7 +18,7 @@ OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed

hadoop-ozone/dist/src/main/compose/ozonefs/docker-config

@@ -19,7 +19,7 @@ OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed

hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config

@@ -18,7 +18,7 @@ OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=true
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.scm.client.address=scm

hadoop-ozone/dist/src/main/compose/ozones3/docker-config

@@ -17,7 +17,7 @@
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=true
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.scm.client.address=scm

hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config

@@ -19,7 +19,7 @@ CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
 OZONE-SITE.XML_ozone.ksm.address=ksm
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=true
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.block.client.address=scm

hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config

@@ -18,7 +18,7 @@ OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed

hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config

@@ -20,7 +20,7 @@ OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed

hadoop-ozone/dist/src/main/compose/ozonetrace/docker-config

@@ -17,7 +17,7 @@
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=true
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.scm.client.address=scm

hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml

@@ -18,9 +18,8 @@ kind: ConfigMap
 metadata:
   name: config
 data:
-
   OZONE-SITE.XML_hdds.datanode.dir: "/data/storage"
-  OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data"
+  OZONE-SITE.XML_ozone.scm.datanode.id: "/data/datanode.id"
   OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata"
   OZONE-SITE.XML_ozone.scm.block.client.address: "scm-0.scm"
   OZONE-SITE.XML_ozone.om.address: "om-0.om"

hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml

@@ -20,7 +20,7 @@ metadata:
   name: config
 data:
   OZONE-SITE.XML_hdds.datanode.dir: /data/storage
-  OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/datanode.id
+  OZONE-SITE.XML_ozone.scm.datanode.id: /data/datanode.id
   OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
   OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm
   OZONE-SITE.XML_ozone.om.address: om-0.om

hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml

@@ -20,7 +20,7 @@ metadata:
   name: config
 data:
   OZONE-SITE.XML_hdds.datanode.dir: /data/storage
-  OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/datanode.id
+  OZONE-SITE.XML_ozone.scm.datanode.id: /data/datanode.id
   OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
   OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm
   OZONE-SITE.XML_ozone.om.address: om-0.om

hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml

@@ -20,7 +20,7 @@ metadata:
   name: config
 data:
   OZONE-SITE.XML_hdds.datanode.dir: /data/storage
-  OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/datanode.id
+  OZONE-SITE.XML_ozone.scm.datanode.id: /data/datanode.id
   OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
   OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm
   OZONE-SITE.XML_ozone.om.address: om-0.om