HDFS-11743. Revert the incompatible fsck reporting output in HDFS-7933 from branch-2.7.

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -349,6 +349,9 @@ Release 2.7.4 - UNRELEASED
     HDFS-11708. Positional read will fail if replicas moved to different DNs
     after stream is opened. (vinayakumarb)
 
+    HDFS-11743. Revert the incompatible fsck reporting output in HDFS-7933
+    from branch-2.7. (zhz)
+
 Release 2.7.3 - 2016-08-25
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java

@@ -245,10 +245,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
       out.println("No. of live Replica: " + numberReplicas.liveReplicas());
       out.println("No. of excess Replica: " + numberReplicas.excessReplicas());
       out.println("No. of stale Replica: " + numberReplicas.replicasOnStaleNodes());
-      out.println("No. of decommissioned Replica: "
+      out.println("No. of decommission Replica: "
           + numberReplicas.decommissioned());
-      out.println("No. of decommissioning Replica: "
-          + numberReplicas.decommissioning());
       out.println("No. of corrupted Replica: " + numberReplicas.corruptReplicas());
       //record datanodes that have corrupted block replica
       Collection<DatanodeDescriptor> corruptionRecord = null;
@@ -511,16 +509,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
       NumberReplicas numberReplicas =
           namenode.getNamesystem().getBlockManager().countNodes(block.getLocalBlock());
       int liveReplicas = numberReplicas.liveReplicas();
-      int decommissionedReplicas = numberReplicas.decommissioned();;
-      int decommissioningReplicas = numberReplicas.decommissioning();
-      res.decommissionedReplicas +=  decommissionedReplicas;
-      res.decommissioningReplicas += decommissioningReplicas;
-      int totalReplicas = liveReplicas + decommissionedReplicas +
-          decommissioningReplicas;
-      res.totalReplicas += totalReplicas;
+      res.totalReplicas += liveReplicas;
       short targetFileReplication = file.getReplication();
       res.numExpectedReplicas += targetFileReplication;
-      if(totalReplicas < res.minReplication){
+      if(liveReplicas < res.minReplication){
         res.numUnderMinReplicatedBlocks++;
       }
       if (liveReplicas > targetFileReplication) {
@@ -540,10 +532,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
         out.print("\n" + path + ": CORRUPT blockpool " + block.getBlockPoolId() + 
             " block " + block.getBlockName()+"\n");
       }
-      if (totalReplicas >= res.minReplication)
+      if (liveReplicas >= res.minReplication)
         res.numMinReplicatedBlocks++;
-      if (totalReplicas < targetFileReplication && totalReplicas > 0) {
-        res.missingReplicas += (targetFileReplication - totalReplicas);
+      if (liveReplicas < targetFileReplication && liveReplicas > 0) {
+        res.missingReplicas += (targetFileReplication - liveReplicas);
         res.numUnderReplicatedBlocks += 1;
         underReplicatedPerFile++;
         if (!showFiles) {
@@ -552,9 +544,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
         out.println(" Under replicated " + block +
                     ". Target Replicas is " +
                     targetFileReplication + " but found " +
-                    liveReplicas + " live replica(s), " +
-                    decommissionedReplicas + " decommissioned replica(s) and " +
-                    decommissioningReplicas + " decommissioning replica(s).");
+                    liveReplicas + " replica(s).");
       }
 
       // count mis replicated blocks
@@ -572,7 +562,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
                     block + ". " + blockPlacementStatus.getErrorDescription());
       }
       report.append(i + ". " + blkName + " len=" + block.getNumBytes());
-      if (totalReplicas == 0) {
+      if (liveReplicas == 0) {
         report.append(" MISSING!");
         res.addMissing(block.toString(), block.getNumBytes());
         missing++;
@@ -872,8 +862,6 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
     long corruptBlocks = 0L;
     long excessiveReplicas = 0L;
     long missingReplicas = 0L;
-    long decommissionedReplicas = 0L;
-    long decommissioningReplicas = 0L;
     long numUnderMinReplicatedBlocks=0L;
     long numOverReplicatedBlocks = 0L;
     long numUnderReplicatedBlocks = 0L;
@@ -945,7 +933,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
         res.append(" (Total open file blocks (not validated): ").append(
             totalOpenFilesBlocks).append(")");
       }
-      if (corruptFiles > 0 || numUnderMinReplicatedBlocks > 0) {
+      if (corruptFiles > 0 || numUnderMinReplicatedBlocks>0) {
         res.append("\n  ********************************");
         if(numUnderMinReplicatedBlocks>0){
           res.append("\n  UNDER MIN REPL'D BLOCKS:\t").append(numUnderMinReplicatedBlocks);
@@ -1008,14 +996,6 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
             ((float) (missingReplicas * 100) / (float) numExpectedReplicas)).append(
             " %)");
       }
-      if (decommissionedReplicas > 0) {
-        res.append("\n DecommissionedReplicas:\t").append(
-            decommissionedReplicas);
-      }
-      if (decommissioningReplicas > 0) {
-        res.append("\n DecommissioningReplicas:\t").append(
-            decommissioningReplicas);
-      }
       return res.toString();
     }
   }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientReportBadBlock.java

@@ -191,7 +191,7 @@ public class TestClientReportBadBlock {
       verifyFirstBlockCorrupted(filePath, false);
       int expectedReplicaCount = repl-corruptBlocReplicas;
       verifyCorruptedBlockCount(filePath, expectedReplicaCount);
-      verifyFsckHealth("Target Replicas is 3 but found 1 live replica");
+      verifyFsckHealth("Target Replicas is 3 but found 1 replica");
       testFsckListCorruptFilesBlocks(filePath, 0);
     }
   }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java

@@ -1471,122 +1471,4 @@ public class TestFsck {
       }
     }
   }
-
-  /**
-   * Test for blocks on decommissioning hosts are not shown as missing
-   */
-  @Test
-  public void testFsckWithDecommissionedReplicas() throws Exception {
-
-    final short REPL_FACTOR = 1;
-    short NUM_DN = 2;
-    final long blockSize = 512;
-    final long fileSize = 1024;
-    boolean checkDecommissionInProgress = false;
-    String [] racks = {"/rack1", "/rack2"};
-    String [] hosts = {"host1", "host2"};
-
-    Configuration conf = new Configuration();
-    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
-    conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
-
-    MiniDFSCluster cluster;
-    DistributedFileSystem dfs ;
-    cluster =
-        new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts)
-            .racks(racks).build();
-
-    assertNotNull("Failed Cluster Creation", cluster);
-    cluster.waitClusterUp();
-    dfs = cluster.getFileSystem();
-    assertNotNull("Failed to get FileSystem", dfs);
-
-    DFSTestUtil util = new DFSTestUtil.Builder().
-        setName(getClass().getSimpleName()).setNumFiles(1).build();
-
-    //create files
-    final String testFile = new String("/testfile");
-    final Path path = new Path(testFile);
-    util.createFile(dfs, path, fileSize, REPL_FACTOR, 1000L);
-    util.waitReplication(dfs, path, REPL_FACTOR);
-    try {
-      // make sure datanode that has replica is fine before decommission
-      String outStr = runFsck(conf, 0, true, testFile);
-      System.out.println(outStr);
-      assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
-
-      // decommission datanode
-      ExtendedBlock eb = util.getFirstBlock(dfs, path);
-      DatanodeDescriptor dn = cluster.getNameNode().getNamesystem()
-          .getBlockManager().getBlockCollection(eb.getLocalBlock())
-          .getBlocks()[0].getDatanode(0);
-      cluster.getNameNode().getNamesystem().getBlockManager()
-          .getDatanodeManager().getDecomManager().startDecommission(dn);
-      String dnName = dn.getXferAddr();
-
-      // wait for decommission start
-      DatanodeInfo datanodeInfo = null;
-      int count = 0;
-      do {
-        Thread.sleep(2000);
-        for (DatanodeInfo info : dfs.getDataNodeStats()) {
-          if (dnName.equals(info.getXferAddr())) {
-            datanodeInfo = info;
-          }
-        }
-        // check the replica status should be healthy(0)
-        // instead of corruption (1) during decommissioning
-        if(!checkDecommissionInProgress && datanodeInfo != null
-            && datanodeInfo.isDecommissionInProgress()) {
-          String fsckOut = runFsck(conf, 0, true, testFile);
-          checkDecommissionInProgress =  true;
-        }
-      } while (datanodeInfo != null && !datanodeInfo.isDecommissioned());
-
-      // check the replica status should be healthy(0) after decommission
-      // is done
-      String fsckOut = runFsck(conf, 0, true, testFile);
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
-    }
-  }
-
-  @Test(timeout = 300000)
-  public void testFsckCorruptWhenOneReplicaIsCorrupt()
-      throws Exception {
-    Configuration conf = new HdfsConfiguration();
-    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
-        .nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(2)
-        .build();
-    try {
-      cluster.waitActive();
-      FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf);
-      cluster.transitionToActive(0);
-      String filePath = "/appendTest";
-      Path fileName = new Path(filePath);
-      DFSTestUtil.createFile(fs, fileName, 512, (short) 2, 0);
-      DFSTestUtil.waitReplication(fs, fileName, (short) 2);
-      assertTrue("File not created", fs.exists(fileName));
-      cluster.getDataNodes().get(1).shutdown();
-      DFSTestUtil.appendFile(fs, fileName, "appendCorruptBlock");
-      cluster.restartDataNode(1, true);
-      GenericTestUtils.waitFor(new Supplier<Boolean>() {
-        @Override
-        public Boolean get() {
-          return (
-              cluster.getNameNode(0).getNamesystem().getCorruptReplicaBlocks()
-                  > 0);
-        }
-      }, 100, 5000);
-
-      DFSTestUtil.appendFile(fs, fileName, "appendCorruptBlock");
-      runFsck(cluster.getConfiguration(0), 0, true, "/");
-    }finally {
-      if(cluster!=null){
-        cluster.shutdown();
-      }
-    }
-  }
 }