HDDS-919. Enable prometheus endpoints for Ozone datanodes (#502)

Pull request from Elek, Márton. (#502)

(cherry picked from commit 7f636b47b5e430e340c8fd731fd9e35e22f9ddbf)

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java

@@ -227,4 +227,24 @@ public final class HddsConfigKeys {
   public static final String HDDS_SECURITY_CLIENT_SCM_CERTIFICATE_PROTOCOL_ACL =
       "hdds.security.client.scm.certificate.protocol.acl";
 
+  public static final String HDDS_DATANODE_HTTP_ENABLED_KEY =
+      "hdds.datanode.http.enabled";
+  public static final String HDDS_DATANODE_HTTP_BIND_HOST_KEY =
+      "hdds.datanode.http-bind-host";
+  public static final String HDDS_DATANODE_HTTPS_BIND_HOST_KEY =
+      "hdds.datanode.https-bind-host";
+  public static final String HDDS_DATANODE_HTTP_ADDRESS_KEY =
+      "hdds.datanode.http-address";
+  public static final String HDDS_DATANODE_HTTPS_ADDRESS_KEY =
+      "hdds.datanode.https-address";
+
+  public static final String HDDS_DATANODE_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
+  public static final int HDDS_DATANODE_HTTP_BIND_PORT_DEFAULT = 9882;
+  public static final int HDDS_DATANODE_HTTPS_BIND_PORT_DEFAULT = 9883;
+  public static final String
+      HDDS_DATANODE_HTTP_KERBEROS_PRINCIPAL_KEY =
+      "hdds.datanode.http.kerberos.principal";
+  public static final String
+      HDDS_DATANODE_HTTP_KERBEROS_KEYTAB_FILE_KEY =
+      "hdds.datanode.http.kerberos.keytab";
 }

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -2029,4 +2029,109 @@
     </description>
   </property>
 
+  <property>
+    <name>hdds.datanode.http.kerberos.principal</name>
+    <value>HTTP/_HOST@EXAMPLE.COM</value>
+    <tag>HDDS, SECURITY, MANAGEMENT</tag>
+    <description>
+      The kerberos principal for the datanode http server.
+    </description>
+  </property>
+  <property>
+    <name>hdds.datanode.http.kerberos.keytab</name>
+    <value>/etc/security/keytabs/HTTP.keytab</value>
+    <tag>HDDS, SECURITY, MANAGEMENT</tag>
+    <description>
+      The kerberos keytab file for datanode http server
+    </description>
+  </property>
+  <property>
+    <name>hdds.datanode.http-address</name>
+    <value>0.0.0.0:9882</value>
+    <tag>HDDS, MANAGEMENT</tag>
+    <description>
+      The address and the base port where the Datanode web ui will listen on.
+      If the port is 0 then the server will start on a free port.
+    </description>
+  </property>
+  <property>
+    <name>hdds.datanode.http-bind-host</name>
+    <value>0.0.0.0</value>
+    <tag>HDDS, MANAGEMENT</tag>
+    <description>
+      The actual address the Datanode web server will bind to. If this
+      optional address is set, it overrides only the hostname portion of
+      hdds.datanode.http-address.
+    </description>
+  </property>
+  <property>
+    <name>hdds.datanode.http.enabled</name>
+    <value>true</value>
+    <tag>HDDS, MANAGEMENT</tag>
+    <description>
+      Property to enable or disable Datanode web ui.
+    </description>
+  </property>
+  <property>
+    <name>hdds.datanode.https-address</name>
+    <value>0.0.0.0:9883</value>
+    <tag>HDDS, MANAGEMENT, SECURITY</tag>
+    <description>
+      The address and the base port where the Datanode web UI will listen
+      on using HTTPS.
+
+      If the port is 0 then the server will start on a free port.
+    </description>
+  </property>
+  <property>
+    <name>hdds.datanode.https-bind-host</name>
+    <value>0.0.0.0</value>
+    <tag>HDDS, MANAGEMENT, SECURITY</tag>
+    <description>
+      The actual address the Datanode web server will bind to using HTTPS.
+      If this optional address is set, it overrides only the hostname portion of
+      hdds.datanode.http-address.
+    </description>
+  </property>
+  <property>
+    <name>ozone.client.retry.max.attempts</name>
+    <value>10</value>
+    <description>
+      Max retry attempts for Ozone RpcClient talking to OzoneManagers.
+    </description>
+  </property>
+  <property>
+    <name>ozone.client.failover.max.attempts</name>
+    <value>15</value>
+    <description>
+      Expert only. The number of client failover attempts that should be
+      made before the failover is considered failed.
+    </description>
+  </property>
+  <property>
+    <name>ozone.client.failover.sleep.base.millis</name>
+    <value>500</value>
+    <description>
+      Expert only. The time to wait, in milliseconds, between failover
+      attempts increases exponentially as a function of the number of
+      attempts made so far, with a random factor of +/- 50%. This option
+      specifies the base value used in the failover calculation. The
+      first failover will retry immediately. The 2nd failover attempt
+      will delay at least ozone.client.failover.sleep.base.millis
+      milliseconds. And so on.
+    </description>
+  </property>
+  <property>
+    <name>ozone.client.failover.sleep.max.millis</name>
+    <value>15000</value>
+    <description>
+      Expert only. The time to wait, in milliseconds, between failover
+      attempts increases exponentially as a function of the number of
+      attempts made so far, with a random factor of +/- 50%. This option
+      specifies the maximum value to wait between failovers.
+      Specifically, the time between two failover attempts will not
+      exceed +/- 50% of ozone.client.failover.sleep.max.millis
+      milliseconds.
+    </description>
+  </property>
 </configuration>

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeHttpServer.java

@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.server.BaseHttpServer;
+
+/**
+ * Simple http server to provide basic monitoring for hdds datanode.
+ * <p>
+ * This server is used to access default /conf /prom /prof endpoints.
+ */
+public class HddsDatanodeHttpServer extends BaseHttpServer {
+
+  public HddsDatanodeHttpServer(Configuration conf) throws IOException {
+    super(conf, "hddsDatanode");
+  }
+
+  @Override
+  protected String getHttpAddressKey() {
+    return HddsConfigKeys.HDDS_DATANODE_HTTP_ADDRESS_KEY;
+  }
+
+  @Override
+  protected String getHttpBindHostKey() {
+    return HddsConfigKeys.HDDS_DATANODE_HTTP_BIND_HOST_KEY;
+  }
+
+  @Override
+  protected String getHttpsAddressKey() {
+    return HddsConfigKeys.HDDS_DATANODE_HTTPS_ADDRESS_KEY;
+  }
+
+  @Override
+  protected String getHttpsBindHostKey() {
+    return HddsConfigKeys.HDDS_DATANODE_HTTPS_BIND_HOST_KEY;
+  }
+
+  @Override
+  protected String getBindHostDefault() {
+    return HddsConfigKeys.HDDS_DATANODE_HTTP_BIND_HOST_DEFAULT;
+  }
+
+  @Override
+  protected int getHttpBindPortDefault() {
+    return HddsConfigKeys.HDDS_DATANODE_HTTP_BIND_PORT_DEFAULT;
+  }
+
+  @Override
+  protected int getHttpsBindPortDefault() {
+    return HddsConfigKeys.HDDS_DATANODE_HTTPS_BIND_PORT_DEFAULT;
+  }
+
+  @Override
+  protected String getKeytabFile() {
+    return HddsConfigKeys.HDDS_DATANODE_HTTP_KERBEROS_KEYTAB_FILE_KEY;
+  }
+
+  @Override
+  protected String getSpnegoPrincipal() {
+    return HddsConfigKeys.HDDS_DATANODE_HTTP_KERBEROS_PRINCIPAL_KEY;
+  }
+
+  @Override
+  protected String getEnabledKey() {
+    return HddsConfigKeys.HDDS_DATANODE_HTTP_ENABLED_KEY;
+  }
+}

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java

@@ -19,6 +19,7 @@ package org.apache.hadoop.ozone;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.HddsUtils;
@@ -67,6 +68,7 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
   private DatanodeDetails datanodeDetails;
   private DatanodeStateMachine datanodeStateMachine;
   private List<ServicePlugin> plugins;
+  private HddsDatanodeHttpServer httpServer;
 
   /**
    * Default constructor.
@@ -180,6 +182,12 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
           LOG.info("Hdds Datanode login successful.");
         }
         datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf);
+        try {
+          httpServer = new HddsDatanodeHttpServer(conf);
+          httpServer.start();
+        } catch (Exception ex) {
+          LOG.error("HttpServer failed to start.", ex);
+        }
         startPlugins();
         // Starting HDDS Daemons
         datanodeStateMachine.startDaemon();
@@ -294,6 +302,14 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
     if (datanodeStateMachine != null) {
       datanodeStateMachine.stopDaemon();
     }
+    if (httpServer != null) {
+      try {
+        httpServer.stop();
+      } catch (Exception e) {
+        LOG.error("Stopping HttpServer is failed.", e);
+      }
+    }
+
   }
 
   @Override

hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/.gitkeep

@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */

hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml

@@ -23,6 +23,7 @@ services:
         - ../..:/opt/hadoop
       ports:
         - 9864
+        - 9882
       command: ["/opt/hadoop/bin/ozone","datanode"]
       env_file:
         - ./docker-config

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

@@ -577,6 +577,7 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
 
     private void configureHddsDatanodes() {
       conf.set(ScmConfigKeys.HDDS_REST_HTTP_ADDRESS_KEY, "0.0.0.0:0");
+      conf.set(HddsConfigKeys.HDDS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
       conf.set(HDDS_DATANODE_PLUGINS_KEY,
           "org.apache.hadoop.ozone.web.OzoneHddsDatanodeService");
       conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT,