HADOOP-1855. fsck verifies block placement policies and reports

violations.  (Konstantin Shvachko via dhruba)



git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@586262 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -11,6 +11,9 @@ Trunk (unreleased changes)
     HADOOP-2058.  Facilitate creating new datanodes dynamically in
     MiniDFSCluster. (Hairong Kuang via dhruba)
 
+    HADOOP-1855.  fsck verifies block placement policies and reports
+    violations.  (Konstantin Shvachko via dhruba)
+
 Branch 0.15 (unreleased changes)
 
   INCOMPATIBLE CHANGES

src/java/org/apache/hadoop/dfs/BlockCrcUpgrade.java

@@ -1220,7 +1220,7 @@ class BlockCrcUpgradeUtils {
     
     public BlockCrcUpgradeStatusReport(int version, short status,
                                        String extraText) {
-      super(version, status);
+      super(version, status, false);
       this.extraText = extraText;
     }
     

src/java/org/apache/hadoop/dfs/DFSAdmin.java

@@ -292,7 +292,7 @@ public class DFSAdmin extends FsShell {
     DistributedFileSystem dfs = (DistributedFileSystem) fs;
     UpgradeStatusReport status = dfs.distributedUpgradeProgress(action);
     String statusText = (status == null ? 
-        "There are no distributed upgrades in progress." :
+        "There are no upgrades in progress." :
           status.getStatusText(action == UpgradeAction.DETAILED_STATUS));
     System.out.println(statusText);
     return 0;

src/java/org/apache/hadoop/dfs/FSConstants.java

@@ -135,7 +135,7 @@ public interface FSConstants {
   public enum SafeModeAction{ SAFEMODE_LEAVE, SAFEMODE_ENTER, SAFEMODE_GET; }
 
   // Startup options
-  public enum StartupOption{ FORMAT, REGULAR, UPGRADE, ROLLBACK; }
+  public enum StartupOption{ FORMAT, REGULAR, UPGRADE, ROLLBACK, FINALIZE; }
 
   // type of the datanode report
   public static enum DatanodeReportType {ALL, LIVE, DEAD }

src/java/org/apache/hadoop/dfs/FSImage.java

@@ -32,7 +32,6 @@ import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
-import java.util.Map;
 import java.util.Properties;
 import java.util.Random;
 import java.lang.Math;
@@ -340,12 +339,16 @@ class FSImage extends Storage {
 
   private void doFinalize(StorageDirectory sd) throws IOException {
     File prevDir = sd.getPreviousDir();
-    if (!prevDir.exists())
-      return; // already discarded
+    if (!prevDir.exists()) { // already discarded
+      LOG.info("Directory " + prevDir + " does not exist.");
+      LOG.info("Finalize upgrade for " + sd.root + " is not required.");
+      return;
+    }
     LOG.info("Finalizing upgrade for storage directory " 
-             + sd.root 
-             + ".\n   cur LV = " + this.getLayoutVersion()
-             + "; cur CTime = " + this.getCTime());
+             + sd.root + "."
+             + (getLayoutVersion()==0 ? "" :
+                   "\n   cur LV = " + this.getLayoutVersion()
+                   + "; cur CTime = " + this.getCTime()));
     assert sd.getCurrentDir().exists() : "Current directory must exist.";
     final File tmpDir = sd.getFinalizedTmp();
     // rename previous to tmp and remove

src/java/org/apache/hadoop/dfs/JspHelper.java

@@ -32,6 +32,7 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.jsp.JspWriter;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.dfs.FSConstants.UpgradeAction;
 import org.apache.hadoop.fs.Path;
 
 public class JspHelper {
@@ -169,7 +170,21 @@ public class JspHelper {
       return "";
     return "Safe mode is ON. <em>" + fsn.getSafeModeTip() + "</em><br>";
   }
-    
+
+  public String getUpgradeStatusText() {
+    String statusText = "";
+    try {
+      UpgradeStatusReport status = 
+        fsn.distributedUpgradeProgress(UpgradeAction.GET_STATUS);
+      statusText = (status == null ? 
+          "There are no upgrades in progress." :
+            status.getStatusText(false));
+    } catch(IOException e) {
+      statusText = "Upgrade status unknown.";
+    }
+    return statusText;
+  }
+
   public void sortNodeList(ArrayList<DatanodeDescriptor> nodes,
                            String field, String order) {
         

src/java/org/apache/hadoop/dfs/NameNode.java

@@ -742,9 +742,30 @@ public class NameNode implements ClientProtocol, DatanodeProtocol,
     return false;
   }
 
+  private static boolean finalize(Configuration conf,
+                               boolean isConfirmationNeeded
+                               ) throws IOException {
+    Collection<File> dirsToFormat = FSNamesystem.getNamespaceDirs(conf);
+    FSNamesystem nsys = new FSNamesystem(new FSImage(dirsToFormat), conf);
+    System.err.print(
+        "\"finalize\" will remove the previous state of the files system.\n"
+        + "Recent upgrade will become permanent.\n"
+        + "Rollback option will not be available anymore.\n");
+    if (isConfirmationNeeded) {
+      System.err.print("Finalize filesystem state ? (Y or N) ");
+      if (!(System.in.read() == 'Y')) {
+        System.err.println("Finalize aborted.");
+        return true;
+      }
+      while(System.in.read() != '\n'); // discard the enter-key
+    }
+    nsys.dir.fsImage.finalizeUpgrade();
+    return false;
+  }
+
   private static void printUsage() {
     System.err.println(
-                       "Usage: java NameNode [-format] | [-upgrade] | [-rollback]");
+      "Usage: java NameNode [-format] | [-upgrade] | [-rollback] | [-finalize]");
   }
 
   private static StartupOption parseArguments(String args[], 
@@ -761,6 +782,8 @@ public class NameNode implements ClientProtocol, DatanodeProtocol,
         startOpt = StartupOption.UPGRADE;
       } else if ("-rollback".equalsIgnoreCase(cmd)) {
         startOpt = StartupOption.ROLLBACK;
+      } else if ("-finalize".equalsIgnoreCase(cmd)) {
+        startOpt = StartupOption.FINALIZE;
       } else
         return null;
     }
@@ -786,12 +809,17 @@ public class NameNode implements ClientProtocol, DatanodeProtocol,
       printUsage();
       return null;
     }
-      
-    if (startOpt == StartupOption.FORMAT) {
-      boolean aborted = format(conf, true);
-      System.exit(aborted ? 1 : 0);
+
+    switch (startOpt) {
+      case FORMAT:
+        boolean aborted = format(conf, true);
+        System.exit(aborted ? 1 : 0);
+      case FINALIZE:
+        aborted = finalize(conf, true);
+        System.exit(aborted ? 1 : 0);
+      default:
     }
-      
+
     NameNode namenode = new NameNode(conf);
     return namenode;
   }

src/java/org/apache/hadoop/dfs/UpgradeManagerNamenode.java

@@ -104,10 +104,17 @@ class UpgradeManagerNamenode extends UpgradeManager {
     FSNamesystem.getFSNamesystem().leaveSafeMode(false);
   }
 
-  UpgradeStatusReport distributedUpgradeProgress(FSConstants.UpgradeAction action 
+  UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action 
                                                 ) throws IOException {
-    if(currentUpgrades == null)
-      return null;  // no upgrades are in progress
+    boolean isFinalized = false;
+    if(currentUpgrades == null) { // no upgrades are in progress
+      FSImage fsimage = FSNamesystem.getFSNamesystem().getFSImage();
+      isFinalized = fsimage.isUpgradeFinalized();
+      if(isFinalized) // upgrade is finalized
+        return null;  // nothing to report
+      return new UpgradeStatusReport(fsimage.getLayoutVersion(), 
+                                     (short)101, isFinalized);
+    }
     UpgradeObjectNamenode curUO = (UpgradeObjectNamenode)currentUpgrades.first();
     boolean details = false;
     switch(action) {

src/java/org/apache/hadoop/dfs/UpgradeObject.java

@@ -40,7 +40,7 @@ abstract class UpgradeObject implements Upgradeable {
 
   public UpgradeStatusReport getUpgradeStatusReport(boolean details) 
                                                     throws IOException {
-    return new UpgradeStatusReport(getVersion(), getUpgradeStatus());
+    return new UpgradeStatusReport(getVersion(), getUpgradeStatus(), false);
   }
 
   public int compareTo(Upgradeable o) {

src/java/org/apache/hadoop/dfs/UpgradeStatusReport.java

@@ -34,15 +34,18 @@ import org.apache.hadoop.io.WritableFactory;
 public class UpgradeStatusReport implements Writable {
   protected int version;
   protected short upgradeStatus;
+  protected boolean finalized;
 
   public UpgradeStatusReport() {
     this.version = 0;
     this.upgradeStatus = 0;
+    this.finalized = false;
   }
 
-  public UpgradeStatusReport( int version, short status) {
+  public UpgradeStatusReport(int version, short status, boolean isFinalized) {
     this.version = version;
     this.upgradeStatus = status;
+    this.finalized = isFinalized;
   }
 
   /**
@@ -62,6 +65,14 @@ public class UpgradeStatusReport implements Writable {
     return upgradeStatus;
   }
 
+  /**
+   * Is current upgrade finalized.
+   * @return true if finalized or false otherwise.
+   */
+  public boolean isFinalized() {
+    return this.finalized;
+  }
+
   /**
    * Get upgradeStatus data as a text for reporting.
    * Should be overloaded for a particular upgrade specific upgradeStatus data.
@@ -71,8 +82,12 @@ public class UpgradeStatusReport implements Writable {
    * @return text
    */
   public String getStatusText(boolean details) {
-    return "Distributed upgrade for version " + getVersion() 
-    + " is in progress. Status = " + getUpgradeStatus() + "%";
+    return "Upgrade for version " + getVersion() 
+            + (upgradeStatus<100 ? 
+              " is in progress. Status = " + upgradeStatus + "%" : 
+              " has been completed."
+              + "\nUpgrade is " + (finalized ? "" : "not ")
+              + "finalized.");
   }
 
   /**

src/webapps/dfs/dfshealth.jsp

@@ -230,10 +230,9 @@
 
 <div id="dfstable"> <table>	  
 <tr> <td id="col1"> Started: <td> <%= fsn.getStartTime()%>
-<tr> <td id="col1"> Version: <td> <%= VersionInfo.getVersion()%>,
-                r<%= VersionInfo.getRevision()%>
-<tr> <td id="col1"> Compiled: <td> <%= VersionInfo.getDate()%> by
-                 <%= VersionInfo.getUser()%>
+<tr> <td id="col1"> Version: <td> <%= VersionInfo.getVersion()%>, r<%= VersionInfo.getRevision()%>
+<tr> <td id="col1"> Compiled: <td> <%= VersionInfo.getDate()%> by <%= VersionInfo.getUser()%>
+<tr> <td id="col1"> Upgrades: <td> <%= jspHelper.getUpgradeStatusText()%>
 </table></div><br>				      
 
 <b><a href="/nn_browsedfscontent.jsp">Browse the filesystem</a></b>

src/webapps/static/hadoop.css

@@ -7,6 +7,10 @@ div#dfsnodetable tr#row1, div#dfstable td#col1 {
 	font-weight : bolder;
 }
 
+div#dfstable td#col1 {
+	vertical-align : top;
+}
+
 div#dfstable td#col3 {
 	text-align : right;
 }
@@ -52,7 +56,7 @@ div#dfsnodetable .rowAlt, .headerASC, .headerDSC {
 }
 
 div#dfstable table {
-	width : 40%;
+	white-space : pre;
 }
 
 div#dfsnodetable td, div#dfsnodetable th, div#dfstable td {