HDFS-17641. Change badly distributed blocks metric protobuf requirement to optional (#7274)

Note: This is a follow on pr from #7123 that walks back an incompatible change that was made.

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java

@@ -38,22 +38,21 @@ public final class ECBlockGroupStats {
   private final long missingBlockGroups;
   private final long bytesInFutureBlockGroups;
   private final long pendingDeletionBlocks;
-  private final long badlyDistributedBlocks;
+  private final Long badlyDistributedBlocks;
   private final Long highestPriorityLowRedundancyBlocks;
 
   public ECBlockGroupStats(long lowRedundancyBlockGroups,
       long corruptBlockGroups, long missingBlockGroups,
-      long bytesInFutureBlockGroups, long pendingDeletionBlocks,
-      long badlyDistributedBlocks) {
+      long bytesInFutureBlockGroups, long pendingDeletionBlocks) {
     this(lowRedundancyBlockGroups, corruptBlockGroups, missingBlockGroups,
         bytesInFutureBlockGroups, pendingDeletionBlocks,
-        badlyDistributedBlocks, null);
+        null, null);
   }
 
   public ECBlockGroupStats(long lowRedundancyBlockGroups,
       long corruptBlockGroups, long missingBlockGroups,
       long bytesInFutureBlockGroups, long pendingDeletionBlocks,
-      long badlyDistributedBlocks, Long highestPriorityLowRedundancyBlocks) {
+      Long badlyDistributedBlocks, Long highestPriorityLowRedundancyBlocks) {
     this.lowRedundancyBlockGroups = lowRedundancyBlockGroups;
     this.corruptBlockGroups = corruptBlockGroups;
     this.missingBlockGroups = missingBlockGroups;
@@ -84,7 +83,11 @@ public final class ECBlockGroupStats {
     return pendingDeletionBlocks;
   }
 
-  public long getBadlyDistributedBlocks() {
+  public boolean hasBadlyDistributedBlocks() {
+    return getBadlyDistributedBlocks() != null;
+  }
+
+  public Long getBadlyDistributedBlocks() {
     return badlyDistributedBlocks;
   }
 
@@ -107,8 +110,11 @@ public final class ECBlockGroupStats {
         .append(", BytesInFutureBlockGroups=").append(
             getBytesInFutureBlockGroups())
         .append(", PendingDeletionBlocks=").append(
-            getPendingDeletionBlocks())
-        .append(" , BadlyDistributedBlocks=").append(getBadlyDistributedBlocks());
+            getPendingDeletionBlocks());
+    if(hasBadlyDistributedBlocks()) {
+      statsBuilder.append(", BadlyDistributedBlocks=")
+          .append(getBadlyDistributedBlocks());
+    }
     if (hasHighestPriorityLowRedundancyBlocks()) {
       statsBuilder.append(", HighestPriorityLowRedundancyBlocks=")
           .append(getHighestPriorityLowRedundancyBlocks());
@@ -163,6 +169,7 @@ public final class ECBlockGroupStats {
     long bytesInFutureBlockGroups = 0;
     long pendingDeletionBlocks = 0;
     long badlyDistributedBlocks = 0;
+    boolean hasBadlyDistributedBlocks = false;
     long highestPriorityLowRedundancyBlocks = 0;
     boolean hasHighestPriorityLowRedundancyBlocks = false;
 
@@ -172,20 +179,22 @@ public final class ECBlockGroupStats {
       missingBlockGroups += stat.getMissingBlockGroups();
       bytesInFutureBlockGroups += stat.getBytesInFutureBlockGroups();
       pendingDeletionBlocks += stat.getPendingDeletionBlocks();
-      badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
+      if (stat.hasBadlyDistributedBlocks()) {
+        hasBadlyDistributedBlocks = true;
+        badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
+      }
       if (stat.hasHighestPriorityLowRedundancyBlocks()) {
         hasHighestPriorityLowRedundancyBlocks = true;
         highestPriorityLowRedundancyBlocks +=
             stat.getHighestPriorityLowRedundancyBlocks();
       }
     }
-    if (hasHighestPriorityLowRedundancyBlocks) {
+    if (hasBadlyDistributedBlocks && hasHighestPriorityLowRedundancyBlocks) {
       return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
           missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
           badlyDistributedBlocks, highestPriorityLowRedundancyBlocks);
     }
     return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
-        missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
-        badlyDistributedBlocks);
+        missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks);
   }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReplicatedBlockStats.java

@@ -37,22 +37,22 @@ public final class ReplicatedBlockStats {
   private final long missingReplicationOneBlocks;
   private final long bytesInFutureBlocks;
   private final long pendingDeletionBlocks;
-  private final long badlyDistributedBlocks;
+  private final Long badlyDistributedBlocks;
   private final Long highestPriorityLowRedundancyBlocks;
 
   public ReplicatedBlockStats(long lowRedundancyBlocks,
       long corruptBlocks, long missingBlocks,
       long missingReplicationOneBlocks, long bytesInFutureBlocks,
-      long pendingDeletionBlocks, long badlyDistributedBlocks) {
+      long pendingDeletionBlocks) {
     this(lowRedundancyBlocks, corruptBlocks, missingBlocks,
         missingReplicationOneBlocks, bytesInFutureBlocks, pendingDeletionBlocks,
-        badlyDistributedBlocks, null);
+        null, null);
   }
 
   public ReplicatedBlockStats(long lowRedundancyBlocks,
       long corruptBlocks, long missingBlocks,
       long missingReplicationOneBlocks, long bytesInFutureBlocks,
-      long pendingDeletionBlocks, long badlyDistributedBlocks,
+      long pendingDeletionBlocks, Long badlyDistributedBlocks,
       Long highestPriorityLowRedundancyBlocks) {
     this.lowRedundancyBlocks = lowRedundancyBlocks;
     this.corruptBlocks = corruptBlocks;
@@ -89,7 +89,11 @@ public final class ReplicatedBlockStats {
     return pendingDeletionBlocks;
   }
 
-  public long getBadlyDistributedBlocks() {
+  public boolean hasBadlyDistributedBlocks() {
+    return getBadlyDistributedBlocks() != null;
+  }
+
+  public Long getBadlyDistributedBlocks() {
     return badlyDistributedBlocks;
   }
 
@@ -113,8 +117,10 @@ public final class ReplicatedBlockStats {
             getMissingReplicationOneBlocks())
         .append(", BytesInFutureBlocks=").append(getBytesInFutureBlocks())
         .append(", PendingDeletionBlocks=").append(
-            getPendingDeletionBlocks())
-        .append(" , badlyDistributedBlocks=").append(getBadlyDistributedBlocks());
+            getPendingDeletionBlocks());
+    if (hasBadlyDistributedBlocks()) {
+      statsBuilder.append(" , badlyDistributedBlocks=").append(getBadlyDistributedBlocks());
+    }
     if (hasHighestPriorityLowRedundancyBlocks()) {
         statsBuilder.append(", HighestPriorityLowRedundancyBlocks=").append(
             getHighestPriorityLowRedundancyBlocks());
@@ -137,6 +143,7 @@ public final class ReplicatedBlockStats {
     long bytesInFutureBlocks = 0;
     long pendingDeletionBlocks = 0;
     long badlyDistributedBlocks = 0;
+    boolean hasBadlyDistributedBlocks = false;
     long highestPriorityLowRedundancyBlocks = 0;
     boolean hasHighestPriorityLowRedundancyBlocks = false;
 
@@ -148,20 +155,24 @@ public final class ReplicatedBlockStats {
       missingReplicationOneBlocks += stat.getMissingReplicationOneBlocks();
       bytesInFutureBlocks += stat.getBytesInFutureBlocks();
       pendingDeletionBlocks += stat.getPendingDeletionBlocks();
-      badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
+
+      if (stat.hasBadlyDistributedBlocks()) {
+        hasBadlyDistributedBlocks = true;
+        badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
+      }
       if (stat.hasHighestPriorityLowRedundancyBlocks()) {
         hasHighestPriorityLowRedundancyBlocks = true;
         highestPriorityLowRedundancyBlocks +=
             stat.getHighestPriorityLowRedundancyBlocks();
       }
     }
-    if (hasHighestPriorityLowRedundancyBlocks) {
+    if (hasBadlyDistributedBlocks && hasHighestPriorityLowRedundancyBlocks) {
       return new ReplicatedBlockStats(lowRedundancyBlocks, corruptBlocks,
           missingBlocks, missingReplicationOneBlocks, bytesInFutureBlocks,
           pendingDeletionBlocks, badlyDistributedBlocks, highestPriorityLowRedundancyBlocks);
     }
     return new ReplicatedBlockStats(lowRedundancyBlocks, corruptBlocks,
         missingBlocks, missingReplicationOneBlocks, bytesInFutureBlocks,
-        pendingDeletionBlocks, badlyDistributedBlocks);
+        pendingDeletionBlocks);
   }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java

@@ -2033,7 +2033,7 @@ public class PBHelperClient {
 
   public static ReplicatedBlockStats convert(
       GetFsReplicatedBlockStatsResponseProto res) {
-    if (res.hasHighestPrioLowRedundancyBlocks()) {
+    if (res.hasBadlyDistributedBlocks() && res.hasHighestPrioLowRedundancyBlocks()) {
       return new ReplicatedBlockStats(res.getLowRedundancy(),
           res.getCorruptBlocks(), res.getMissingBlocks(),
           res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
@@ -2043,12 +2043,12 @@ public class PBHelperClient {
     return new ReplicatedBlockStats(res.getLowRedundancy(),
         res.getCorruptBlocks(), res.getMissingBlocks(),
         res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
-        res.getBadlyDistributedBlocks(), res.getPendingDeletionBlocks());
+        res.getPendingDeletionBlocks());
   }
 
   public static ECBlockGroupStats convert(
       GetFsECBlockGroupStatsResponseProto res) {
-    if (res.hasHighestPrioLowRedundancyBlocks()) {
+    if (res.hasBadlyDistributedBlocks() && res.hasHighestPrioLowRedundancyBlocks()) {
       return new ECBlockGroupStats(res.getLowRedundancy(),
           res.getCorruptBlocks(), res.getMissingBlocks(),
           res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
@@ -2056,8 +2056,7 @@ public class PBHelperClient {
     }
     return new ECBlockGroupStats(res.getLowRedundancy(),
         res.getCorruptBlocks(), res.getMissingBlocks(),
-        res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
-        res.getBadlyDistributedBlocks());
+        res.getBlocksInFuture(), res.getPendingDeletionBlocks());
   }
 
   public static DatanodeReportTypeProto convert(DatanodeReportType t) {
@@ -2526,8 +2525,10 @@ public class PBHelperClient {
         replicatedBlockStats.getBytesInFutureBlocks());
     result.setPendingDeletionBlocks(
         replicatedBlockStats.getPendingDeletionBlocks());
-    result.setBadlyDistributedBlocks(
-        replicatedBlockStats.getBadlyDistributedBlocks());
+    if (replicatedBlockStats.hasBadlyDistributedBlocks()) {
+      result.setBadlyDistributedBlocks(
+          replicatedBlockStats.getBadlyDistributedBlocks());
+    }
     if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
       result.setHighestPrioLowRedundancyBlocks(
           replicatedBlockStats.getHighestPriorityLowRedundancyBlocks());
@@ -2547,8 +2548,10 @@ public class PBHelperClient {
         ecBlockGroupStats.getBytesInFutureBlockGroups());
     result.setPendingDeletionBlocks(
         ecBlockGroupStats.getPendingDeletionBlocks());
-    result.setBadlyDistributedBlocks(
-        ecBlockGroupStats.getBadlyDistributedBlocks());
+    if (ecBlockGroupStats.hasBadlyDistributedBlocks()) {
+      result.setBadlyDistributedBlocks(
+          ecBlockGroupStats.getBadlyDistributedBlocks());
+    }
     if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
       result.setHighestPrioLowRedundancyBlocks(
           ecBlockGroupStats.getHighestPriorityLowRedundancyBlocks());

hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto

@@ -372,8 +372,7 @@ message GetFsReplicatedBlockStatsResponseProto {
   required uint64 blocks_in_future = 5;
   required uint64 pending_deletion_blocks = 6;
   optional uint64 highest_prio_low_redundancy_blocks = 7;
-  required uint64 badly_distributed_blocks = 8;
-
+  optional uint64 badly_distributed_blocks = 8;
 }
 
 message GetFsECBlockGroupStatsRequestProto { // no input paramters
@@ -386,7 +385,7 @@ message GetFsECBlockGroupStatsResponseProto {
   required uint64 blocks_in_future = 4;
   required uint64 pending_deletion_blocks = 5;
   optional uint64 highest_prio_low_redundancy_blocks = 6;
-  required uint64 badly_distributed_blocks = 7;
+  optional uint64 badly_distributed_blocks = 7;
 }
 
 enum DatanodeReportTypeProto {  // type of the datanode report

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java

@@ -554,8 +554,10 @@ public class DFSAdmin extends FsShell {
         replicatedBlockStats.getMissingReplicaBlocks());
     System.out.println("\tMissing blocks (with replication factor 1): " +
         replicatedBlockStats.getMissingReplicationOneBlocks());
-    System.out.println("\tBadly Distributed Blocks: " +
-        replicatedBlockStats.getBadlyDistributedBlocks());
+    if (replicatedBlockStats.hasBadlyDistributedBlocks()) {
+      System.out.println("\tBadly Distributed Blocks: " +
+          replicatedBlockStats.getBadlyDistributedBlocks());
+    }
     if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
       System.out.println("\tLow redundancy blocks with highest priority " +
           "to recover: " +
@@ -573,8 +575,10 @@ public class DFSAdmin extends FsShell {
         ecBlockGroupStats.getCorruptBlockGroups());
     System.out.println("\tMissing block groups: " +
         ecBlockGroupStats.getMissingBlockGroups());
-    System.out.println("\tBadly Distributed Blocks: " +
-        ecBlockGroupStats.getBadlyDistributedBlocks());
+    if (ecBlockGroupStats.hasBadlyDistributedBlocks()) {
+      System.out.println("\tBadly Distributed Blocks: " +
+          ecBlockGroupStats.getBadlyDistributedBlocks());
+    }
     if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
       System.out.println("\tLow redundancy blocks with highest priority " +
           "to recover: " +