HDFS-6800. Support Datanode layout changes with rolling upgrade. (Contributed by James Thomas)

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -371,6 +371,9 @@ Release 2.6.0 - UNRELEASED
     HDFS-6951. Correctly persist raw namespace xattrs to edit log and fsimage.
     (clamb via wang)
 
+    HDFS-6800. Support Datanode layout changes with rolling upgrade.
+    (James Thomas via Arpit Agarwal)
+
     BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
   
       HDFS-6387. HDFS CLI admin tool for creating & deleting an

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java

@@ -269,7 +269,14 @@ public class BlockPoolSliceStorage extends Storage {
    */
   private void doTransition(DataNode datanode, StorageDirectory sd,
       NamespaceInfo nsInfo, StartupOption startOpt) throws IOException {
-    if (startOpt == StartupOption.ROLLBACK) {
+    if (startOpt == StartupOption.ROLLBACK && sd.getPreviousDir().exists()) {
+      // we will already restore everything in the trash by rolling back to
+      // the previous directory, so we must delete the trash to ensure
+      // that it's not restored by BPOfferService.signalRollingUpgrade()
+      if (!FileUtil.fullyDelete(getTrashRootDir(sd))) {
+        throw new IOException("Unable to delete trash directory prior to " +
+            "restoration of previous directory: " + getTrashRootDir(sd));
+      }
       doRollback(sd, nsInfo); // rollback if applicable
     } else {
       // Restore all the files in the trash. The restored files are retained

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java

@@ -247,10 +247,9 @@ public class DataNode extends Configured
     LogFactory.getLog(DataNode.class.getName() + ".clienttrace");
   
   private static final String USAGE =
-      "Usage: java DataNode [-regular | -rollback | -rollingupgrade rollback]\n" +
+      "Usage: java DataNode [-regular | -rollback]\n" +
       "    -regular                 : Normal DataNode startup (default).\n" +
-      "    -rollback                : Rollback a standard upgrade.\n" +
-      "    -rollingupgrade rollback : Rollback a rolling upgrade operation.\n" +
+      "    -rollback                : Rollback a standard or rolling upgrade.\n" +
       "  Refer to HDFS documentation for the difference between standard\n" +
       "  and rolling upgrades.";
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java

@@ -1071,6 +1071,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
     }
 
     if (nn.getFSImage().isUpgradeFinalized() &&
+        !namesystem.isRollingUpgrade() &&
         !nn.isStandbyState() &&
         noStaleStorages) {
       return new FinalizeCommand(poolId);

hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml

@@ -206,7 +206,7 @@
       <li>Restore the pre-upgrade release in all machines.</li>
       <li>Start <em>NNs</em> with the
         "<a href="#namenode_-rollingUpgrade"><code>-rollingUpgrade rollback</code></a>" option.</li>
-      <li>Start <em>DNs</em> normally.</li>
+      <li>Start <em>DNs</em> with the "<code>-rollback</code>" option.</li>
     </ol></li>
   </ul>