HDFS-4514. Add CLI for supporting snapshot rename, diff report, and snapshottable directory listing. Contributed by Jing Zhao

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2802@1449956 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -2241,6 +2241,19 @@ public abstract class FileSystem extends Configured implements Closeable {
         + " doesn't support createSnapshot");
   }
   
+  /**
+   * Rename a snapshot
+   * @param path The directory path where the snapshot was taken
+   * @param snapshotOldName Old name of the snapshot
+   * @param snapshotNewName New name of the snapshot
+   * @throws IOException
+   */
+  public void renameSnapshot(Path path, String snapshotOldName,
+      String snapshotNewName) throws IOException {
+    throw new UnsupportedOperationException(getClass().getSimpleName()
+        + " doesn't support renameSnapshot");
+  }
+  
   /**
    * Delete a snapshot of a directory
    * @param path  The directory that the to-be-deleted snapshot belongs to

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SnapshotCommands.java

@@ -25,6 +25,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.PathIsNotDirectoryException;
 
+import com.google.common.base.Preconditions;
+
 /**
  * Snapshot related operations
  */
@@ -34,10 +36,12 @@ import org.apache.hadoop.fs.PathIsNotDirectoryException;
 class SnapshotCommands extends FsCommand {
   private final static String CREATE_SNAPSHOT = "createSnapshot";
   private final static String DELETE_SNAPSHOT = "deleteSnapshot";
+  private final static String RENAME_SNAPSHOT = "renameSnapshot";
   
   public static void registerCommands(CommandFactory factory) {
     factory.addClass(CreateSnapshot.class, "-" + CREATE_SNAPSHOT);
     factory.addClass(DeleteSnapshot.class, "-" + DELETE_SNAPSHOT);
+    factory.addClass(RenameSnapshot.class, "-" + RENAME_SNAPSHOT);
   }
   
   /**
@@ -45,10 +49,10 @@ class SnapshotCommands extends FsCommand {
    */
   public static class CreateSnapshot extends FsCommand {
     public static final String NAME = CREATE_SNAPSHOT;
-    public static final String USAGE = "<snapshotName> <snapshotRoot>";
+    public static final String USAGE = "<snapshotDir> <snapshotName>";
     public static final String DESCRIPTION = "Create a snapshot on a directory";
 
-    private static String snapshotName;
+    private String snapshotName;
 
     @Override
     protected void processPath(PathData item) throws IOException {
@@ -62,7 +66,7 @@ class SnapshotCommands extends FsCommand {
       if (args.size() != 2) {
         throw new IOException("args number not 2:" + args.size());
       }
-      snapshotName = args.removeFirst();
+      snapshotName = args.removeLast();
       // TODO: name length check  
 
     }
@@ -85,11 +89,11 @@ class SnapshotCommands extends FsCommand {
    */
   public static class DeleteSnapshot extends FsCommand {
     public static final String NAME = DELETE_SNAPSHOT;
-    public static final String USAGE = "<snapshotName> <snapshotDir>";
+    public static final String USAGE = "<snapshotDir> <snapshotName>";
     public static final String DESCRIPTION = 
         "Delete a snapshot from a directory";
 
-    private static String snapshotName;
+    private String snapshotName;
 
     @Override
     protected void processPath(PathData item) throws IOException {
@@ -103,7 +107,7 @@ class SnapshotCommands extends FsCommand {
       if (args.size() != 2) {
         throw new IOException("args number not 2: " + args.size());
       }
-      snapshotName = args.removeFirst();
+      snapshotName = args.removeLast();
       // TODO: name length check
 
     }
@@ -120,5 +124,49 @@ class SnapshotCommands extends FsCommand {
       sroot.fs.deleteSnapshot(sroot.path, snapshotName);
     }
   }
+  
+  /**
+   * Rename a snapshot
+   */
+  public static class RenameSnapshot extends FsCommand {
+    public static final String NAME = RENAME_SNAPSHOT;
+    public static final String USAGE = "<snapshotDir> <oldName> <newName>";
+    public static final String DESCRIPTION = 
+        "Rename a snapshot from oldName to newName";
+    
+    private String oldName;
+    private String newName;
+    
+    @Override
+    protected void processPath(PathData item) throws IOException {
+      if (!item.stat.isDirectory()) {
+        throw new PathIsNotDirectoryException(item.toString());
+      }
+    }
+
+    @Override
+    protected void processOptions(LinkedList<String> args) throws IOException {
+      if (args.size() != 3) {
+        throw new IOException("args number not 3: " + args.size());
+      }
+      newName = args.removeLast();
+      oldName = args.removeLast();
+      
+      // TODO: new name length check
+    }
+
+    @Override
+    protected void processArguments(LinkedList<PathData> items)
+        throws IOException {
+      super.processArguments(items);
+      if (exitCode != 0) { // check for error collecting paths
+        return;
+      }
+      Preconditions.checkArgument(items.size() == 1);
+      PathData sroot = items.getFirst();
+      sroot.fs.renameSnapshot(sroot.path, oldName, newName);
+    }
+    
+  }
 }
 

hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt

@@ -173,3 +173,6 @@ Branch-2802 Snapshot (Unreleased)
 
   HDFS-4520. Support listing snapshots under a snapshottable directory using ls.
   (Jing Zhao via szetszwo)
+
+  HDFS-4514. Add CLI for supporting snapshot rename, diff report, and
+  snapshottable directory listing.  (Jing Zhao via szetszwo)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -47,6 +47,7 @@ import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.VolumeId;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.client.HdfsAdmin;
 import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -906,21 +907,12 @@ public class DistributedFileSystem extends FileSystem {
     return setSafeMode(SafeModeAction.SAFEMODE_GET, true);
   }
 
-  /**
-   * Allow snapshot on a directory.
-   * 
-   * @param path the directory to be taken snapshots
-   * @throws IOException
-   */
+  /** @see HdfsAdmin#allowSnapshot(String) */
   public void allowSnapshot(String path) throws IOException {
     dfs.allowSnapshot(path);
   }
   
-  /**
-   * Disallow snapshot on a directory.
-   * @param path the snapshottable directory.
-   * @throws IOException on error
-   */
+  /** @see HdfsAdmin#disallowSnapshot(String) */
   public void disallowSnapshot(String path) throws IOException {
     dfs.disallowSnapshot(path);
   }
@@ -931,13 +923,7 @@ public class DistributedFileSystem extends FileSystem {
     dfs.createSnapshot(getPathName(path), snapshotName);
   }
   
-  /**
-   * Rename a snapshot
-   * @param path The directory path where the snapshot was taken
-   * @param snapshotOldName Old name of the snapshot
-   * @param snapshotNewName New name of the snapshot
-   * @throws IOException
-   */
+  @Override
   public void renameSnapshot(Path path, String snapshotOldName,
       String snapshotNewName) throws IOException {
     dfs.renameSnapshot(getPathName(path), snapshotOldName, snapshotNewName);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java

@@ -105,4 +105,20 @@ public class HdfsAdmin {
   public void clearSpaceQuota(Path src) throws IOException {
     dfs.setQuota(src, HdfsConstants.QUOTA_DONT_SET, HdfsConstants.QUOTA_RESET);
   }
+  
+  /**
+   * Allow snapshot on a directory.
+   * @param the path of the directory where snapshots will be taken
+   */
+  public void allowSnapshot(String path) throws IOException {
+    dfs.allowSnapshot(path);
+  }
+  
+  /**
+   * Disallow snapshot on a directory.
+   * @param path of the snapshottable directory.
+   */
+  public void disallowSnapshot(String path) throws IOException {
+    dfs.disallowSnapshot(path);
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java

@@ -106,9 +106,9 @@ public class SnapshotDiffReport {
     public String getRelativePathString() {
       String path = DFSUtil.bytes2String(relativePath);
       if (path.isEmpty()) {
-        return ".";
+        return Path.CUR_DIR;
       } else {
-        return "." + Path.SEPARATOR + path;
+        return Path.CUR_DIR + Path.SEPARATOR + path;
       }
     }
 
@@ -183,7 +183,7 @@ public class SnapshotDiffReport {
         "current directory" : "snapshot " + fromSnapshot;
     String to = toSnapshot == null || toSnapshot.isEmpty() ? "current directory"
         : "snapshot " + toSnapshot;
-    str.append("Diffence between snapshot " + from + " and " + to
+    str.append("Difference between " + from + " and " + to
         + " under directory " + snapshotRoot + ":" + LINE_SEPARATOR);
     for (DiffReportEntry entry : diffList) {
       str.append(entry.toString() + LINE_SEPARATOR);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java

@@ -17,6 +17,10 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
+import java.io.PrintStream;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSUtil;
@@ -85,4 +89,59 @@ public class SnapshottableDirectoryStatus {
     return parentFullPathStr == null ? new Path(dirStatus.getLocalName())
         : new Path(parentFullPathStr, dirStatus.getLocalName());
   }
+  
+  /**
+   * Print a list of {@link SnapshottableDirectoryStatus} out to a given stream.
+   * @param stats The list of {@link SnapshottableDirectoryStatus}
+   * @param out The given stream for printing.
+   */
+  public static void print(SnapshottableDirectoryStatus[] stats, 
+      PrintStream out) {
+    if (stats == null || stats.length == 0) {
+      out.println();
+      return;
+    }
+    int maxRepl = 0, maxLen = 0, maxOwner = 0, maxGroup = 0;
+    int maxSnapshotNum = 0, maxSnapshotQuota = 0;
+    for (SnapshottableDirectoryStatus status : stats) {
+      maxRepl = maxLength(maxRepl, status.dirStatus.getReplication());
+      maxLen = maxLength(maxLen, status.dirStatus.getLen());
+      maxOwner = maxLength(maxOwner, status.dirStatus.getOwner());
+      maxGroup = maxLength(maxGroup, status.dirStatus.getGroup());
+      maxSnapshotNum = maxLength(maxSnapshotNum, status.snapshotNumber);
+      maxSnapshotQuota = maxLength(maxSnapshotQuota, status.snapshotQuota);
+    }
+    
+    StringBuilder fmt = new StringBuilder();
+    fmt.append("%s%s "); // permission string
+    fmt.append("%"  + maxRepl  + "s ");
+    fmt.append((maxOwner > 0) ? "%-" + maxOwner + "s " : "%s");
+    fmt.append((maxGroup > 0) ? "%-" + maxGroup + "s " : "%s");
+    fmt.append("%"  + maxLen   + "s ");
+    fmt.append("%s "); // mod time
+    fmt.append("%"  + maxSnapshotNum  + "s ");
+    fmt.append("%"  + maxSnapshotQuota  + "s ");
+    fmt.append("%s"); // path
+    
+    String lineFormat = fmt.toString();
+    SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
+         
+    for (SnapshottableDirectoryStatus status : stats) {
+      String line = String.format(lineFormat, "d", 
+          status.dirStatus.getPermission(),
+          status.dirStatus.getReplication(),
+          status.dirStatus.getOwner(),
+          status.dirStatus.getGroup(),
+          String.valueOf(status.dirStatus.getLen()),
+          dateFormat.format(new Date(status.dirStatus.getModificationTime())),
+          status.snapshotNumber, status.snapshotQuota, 
+          status.getFullPath().toString()
+      );
+      out.println(line);
+    }
+  }
+
+  private static int maxLength(int n, Object value) {
+    return Math.max(n, String.valueOf(value).length());
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java

@@ -408,7 +408,7 @@ public class DFSAdmin extends FsShell {
 
   /**
    * Allow snapshot on a directory.
-   * Usage: java DFSAdmin -allowSnapshot <snapshotRoot>
+   * Usage: java DFSAdmin -allowSnapshot snapshotDir
    * @param argv List of of command line parameters.
    * @exception IOException
    */
@@ -420,7 +420,7 @@ public class DFSAdmin extends FsShell {
   
   /**
    * Allow snapshot on a directory.
-   * Usage: java DFSAdmin -disallowSnapshot <snapshotRoot>
+   * Usage: java DFSAdmin -disallowSnapshot snapshotDir
    * @param argv List of of command line parameters.
    * @exception IOException
    */
@@ -571,6 +571,8 @@ public class DFSAdmin extends FsShell {
       "\t[-deleteBlockPool datanodehost:port blockpoolId [force]]\n"+
       "\t[-setBalancerBandwidth <bandwidth>]\n" +
       "\t[-fetchImage <local directory>]\n" +
+      "\t[-allowSnapshot <snapshotDir>]\n" +
+      "\t[-disallowSnapshot <snapshotDir>]\n" +
       "\t[-help [cmd]]\n";
 
     String report ="-report: \tReports basic filesystem information and statistics.\n";
@@ -661,6 +663,12 @@ public class DFSAdmin extends FsShell {
       "\tDownloads the most recent fsimage from the Name Node and saves it in" +
       "\tthe specified local directory.\n";
     
+    String allowSnapshot = "-allowSnapshot <snapshotDir>:\n" +
+        "\tAllow snapshots to be taken on a directory.\n";
+    
+    String disallowSnapshot = "-disallowSnapshot <snapshotDir>:\n" +
+        "\tDo not allow snapshots to be taken on a directory any more.\n";
+    
     String help = "-help [cmd]: \tDisplays help for the given command or all commands if none\n" +
       "\t\tis specified.\n";
 
@@ -704,6 +712,10 @@ public class DFSAdmin extends FsShell {
       System.out.println(setBalancerBandwidth);
     } else if ("fetchImage".equals(cmd)) {
       System.out.println(fetchImage);
+    } else if ("allowSnapshot".equalsIgnoreCase(cmd)) {
+      System.out.println(allowSnapshot);
+    } else if ("disallowSnapshot".equalsIgnoreCase(cmd)) {
+      System.out.println(disallowSnapshot);
     } else if ("help".equals(cmd)) {
       System.out.println(help);
     } else {
@@ -728,6 +740,8 @@ public class DFSAdmin extends FsShell {
       System.out.println(deleteBlockPool);
       System.out.println(setBalancerBandwidth);
       System.out.println(fetchImage);
+      System.out.println(allowSnapshot);
+      System.out.println(disallowSnapshot);
       System.out.println(help);
       System.out.println();
       ToolRunner.printGenericCommandUsage(System.out);
@@ -906,10 +920,10 @@ public class DFSAdmin extends FsShell {
           + " [-safemode enter | leave | get | wait]");
     } else if ("-allowSnapshot".equalsIgnoreCase(cmd)) {
       System.err.println("Usage: java DFSAdmin"
-          + " [-allowSnapshot <snapshotRoot>]");
-    } else if ("-disallowsnapshot".equalsIgnoreCase(cmd)) {
+          + " [-allowSnapshot <snapshotDir>]");
+    } else if ("-disallowSnapshot".equalsIgnoreCase(cmd)) {
       System.err.println("Usage: java DFSAdmin"
-          + " [-disallowSnapshot <snapshotRoot>]");
+          + " [-disallowSnapshot <snapshotDir>]");
     } else if ("-saveNamespace".equals(cmd)) {
       System.err.println("Usage: java DFSAdmin"
                          + " [-saveNamespace]");
@@ -969,8 +983,8 @@ public class DFSAdmin extends FsShell {
       System.err.println("Note: Administrative commands can only be run as the HDFS superuser.");
       System.err.println("           [-report]");
       System.err.println("           [-safemode enter | leave | get | wait]"); 
-      System.err.println("           [-allowSnapshot <snapshotRoot>]");
-      System.err.println("           [-disallowSnapshot <snapshotRoot>]");
+      System.err.println("           [-allowSnapshot <snapshotDir>]");
+      System.err.println("           [-disallowSnapshot <snapshotDir>]");
       System.err.println("           [-saveNamespace]");
       System.err.println("           [-rollEdits]");
       System.err.println("           [-restoreFailedStorage true|false|check]");
@@ -1021,7 +1035,7 @@ public class DFSAdmin extends FsShell {
         return exitCode;
       }
     } else if ("-allowSnapshot".equalsIgnoreCase(cmd)) {
-      if (argv.length != 3) {
+      if (argv.length != 2) {
         printUsage(cmd);
         return exitCode;
       }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java

@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.snapshot;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+
+/**
+ * A tool used to list all snapshottable directories that are owned by the 
+ * current user. The tool returns all the snapshottable directories if the user
+ * is a super user.
+ */
+@InterfaceAudience.Private
+public class LsSnapshottableDir {
+  public static void main(String[] argv) throws IOException {
+    String description = "LsSnapshottableDir: \n" +
+        "\tGet the list of snapshottable directories that are owned by the current user.\n" +
+        "\tReturn all the snapshottable directories if the current user is a super user.\n";
+
+    if(argv.length != 0) {
+      System.err.println("Usage: \n" + description);
+      System.exit(1);
+    }
+    
+    Configuration conf = new Configuration();
+    FileSystem fs = FileSystem.get(conf);
+    if (! (fs instanceof DistributedFileSystem)) {
+      System.err.println(
+          "LsSnapshottableDir can only be used in DistributedFileSystem");
+      System.exit(1);
+    }
+    DistributedFileSystem dfs = (DistributedFileSystem) fs;
+    
+    SnapshottableDirectoryStatus[] stats = dfs.getSnapshottableDirListing();
+    SnapshottableDirectoryStatus.print(stats, System.out);
+  }
+
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java

@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.snapshot;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+
+/**
+ * A tool used to get the difference report between two snapshots, or between
+ * a snapshot and the current status of a directory. 
+ * <pre>
+ * Usage: SnapshotDiff snapshotDir from to
+ * For from/to, users can use "." to present the current status, and use 
+ * ".snapshot/snapshot_name" to present a snapshot, where ".snapshot/" can be 
+ * omitted.
+ * </pre>
+ */
+@InterfaceAudience.Private
+public class SnapshotDiff {
+  private static String getSnapshotName(String name) {
+    if (Path.CUR_DIR.equals(name)) { // current directory
+      return "";
+    }
+    if (name.startsWith(HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR)
+        || name.startsWith(Path.SEPARATOR + HdfsConstants.DOT_SNAPSHOT_DIR
+            + Path.SEPARATOR)) {
+      // get the snapshot name
+      int i = name.indexOf(HdfsConstants.DOT_SNAPSHOT_DIR);
+      return name.substring(i + HdfsConstants.DOT_SNAPSHOT_DIR.length() + 1);
+    }
+    return name;
+  }
+  
+  public static void main(String[] argv) throws IOException {
+    String description = "SnapshotDiff <snapshotDir> <from> <to>:\n" +
+    "\tGet the difference between two snapshots, \n" + 
+    "\tor between a snapshot and the current tree of a directory.\n" +
+    "\tFor <from>/<to>, users can use \".\" to present the current status,\n" +
+    "\tand use \".snapshot/snapshot_name\" to present a snapshot,\n" +
+    "\twhere \".snapshot/\" can be omitted\n";
+    
+    if(argv.length != 3) {
+      System.err.println("Usage: \n" + description);
+      System.exit(1);
+    }
+    
+    Configuration conf = new Configuration();
+    FileSystem fs = FileSystem.get(conf);
+    if (! (fs instanceof DistributedFileSystem)) {
+      System.err.println(
+          "SnapshotDiff can only be used in DistributedFileSystem");
+      System.exit(1);
+    }
+    DistributedFileSystem dfs = (DistributedFileSystem) fs;
+    
+    Path snapshotRoot = new Path(argv[0]);
+    String fromSnapshot = getSnapshotName(argv[1]);
+    String toSnapshot = getSnapshotName(argv[2]);
+    SnapshotDiffReport diffReport = dfs.getSnapshotDiffReport(snapshotRoot,
+        fromSnapshot, toSnapshot);
+    System.out.println(diffReport.toString());
+  }
+
+}