HDFS-9732, Improve DelegationTokenIdentifier.toString() for better logging. Contributed by Yongjun Zhang.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java

@@ -232,10 +232,35 @@ extends TokenIdentifier {
   public String toString() {
     StringBuilder buffer = new StringBuilder();
     buffer
-        .append("owner=" + owner + ", renewer=" + renewer + ", realUser="
-            + realUser + ", issueDate=" + issueDate + ", maxDate=" + maxDate
-            + ", sequenceNumber=" + sequenceNumber + ", masterKeyId="
-            + masterKeyId);
+        .append(getKind())
+        .append(" owner=").append(owner)
+        .append(", renewer=").append(renewer)
+        .append(", realUser=").append(realUser)
+        .append(", issueDate=").append(issueDate)
+        .append(", maxDate=").append(maxDate)
+        .append(", sequenceNumber=").append(sequenceNumber)
+        .append(", masterKeyId=").append(masterKeyId);
+    return buffer.toString();
+  }
+  /*
+   * A frozen version of toString() to be used to be backward compatible.
+   * When backward compatibility is not needed, use toString(), which provides
+   * more info and is supposed to evolve, see HDFS-9732.
+   * Don't change this method except for major revisions.
+   *
+   * NOTE:
+   * Currently this method is used by CLI for backward compatibility.
+   */
+  public String toStringStable() {
+    StringBuilder buffer = new StringBuilder();
+    buffer
+        .append("owner=").append(owner)
+        .append(", renewer=").append(renewer)
+        .append(", realUser=").append(realUser)
+        .append(", issueDate=").append(issueDate)
+        .append(", maxDate=").append(maxDate)
+        .append(", sequenceNumber=").append(sequenceNumber)
+        .append(", masterKeyId=").append(masterKeyId);
     return buffer.toString();
   }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java

@@ -60,9 +60,30 @@ public class DelegationTokenIdentifier
 
   @Override
   public String toString() {
-    return getKind() + " token " + getSequenceNumber()
-        + " for " + getUser().getShortUserName() + " with renewer " +
-        getRenewer();
+    StringBuilder sbld = new StringBuilder();
+    sbld
+        .append("token for ").append(getUser().getShortUserName())
+        .append(": ").append(super.toString());
+    return sbld.toString();
+  }
+
+  /*
+   * A frozen version of toString() to be used to be backward compatible.
+   * When backward compatibility is not needed, use toString(), which provides
+   * more info and is supposed to evolve, see HDFS-9732.
+   * Don't change this method except for major revisions.
+   *
+   * NOTE:
+   * Currently this method is used by CLI for backward compatibility.
+   */
+  @Override
+  public String toStringStable() {
+    StringBuilder sbld = new StringBuilder();
+    sbld
+        .append(getKind()).append(" token ").append(getSequenceNumber())
+        .append(" for ").append(getUser().getShortUserName())
+        .append(" with renewer ").append(getRenewer());
+    return sbld.toString();
   }
 
   /** @return a string representation of the token */

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java

@@ -62,7 +62,7 @@ public class DelegationTokenFetcher {
   private static final String PRINT = "print";
   private static final String RENEW = "renew";
   private static final String RENEWER = "renewer";
-
+  private static final String VERBOSE = "verbose";
   /**
    * Command-line interface
    */
@@ -75,6 +75,7 @@ public class DelegationTokenFetcher {
       .addOption(CANCEL, false, "cancel the token")
       .addOption(RENEW, false, "renew the token")
       .addOption(PRINT, false, "print the token")
+      .addOption(VERBOSE, false, "print verbose output")
       .addOption(HELP_SHORT, HELP, false, "print out help information");
 
     GenericOptionsParser parser = new GenericOptionsParser(conf,
@@ -88,6 +89,7 @@ public class DelegationTokenFetcher {
     final boolean cancel = cmd.hasOption(CANCEL);
     final boolean renew = cmd.hasOption(RENEW);
     final boolean print = cmd.hasOption(PRINT);
+    final boolean verbose = cmd.hasOption(VERBOSE);
     final boolean help = cmd.hasOption(HELP);
     String[] remaining = parser.getRemainingArgs();
 
@@ -115,7 +117,7 @@ public class DelegationTokenFetcher {
       @Override
       public Object run() throws Exception {
         if (print) {
-          printTokens(conf, tokenFile);
+          printTokens(conf, tokenFile, verbose);
         } else if (cancel) {
           cancelTokens(conf, tokenFile);
         } else if (renew) {
@@ -191,17 +193,32 @@ public class DelegationTokenFetcher {
     }
   }
 
-  private static void printTokens(final Configuration conf,
-                                  final Path tokenFile)
-          throws IOException {
+  @VisibleForTesting
+  static String printTokensToString(
+      final Configuration conf,
+      final Path tokenFile,
+      final boolean verbose) throws IOException {
+    StringBuilder sbld = new StringBuilder();
+    final String nl = System.getProperty("line.separator");
     DelegationTokenIdentifier id = new DelegationTokenSecretManager(0, 0, 0,
             0, null).createIdentifier();
     for (Token<?> token : readTokens(tokenFile, conf)) {
       DataInputStream in = new DataInputStream(new ByteArrayInputStream(token
               .getIdentifier()));
       id.readFields(in);
-      System.out.println("Token (" + id + ") for " + token.getService());
+      String idStr = (verbose? id.toString() : id.toStringStable());
+      sbld
+          .append("Token (").append(idStr)
+          .append(") for ").append(token.getService()).append(nl);
     }
+    return sbld.toString();
+  }
+
+  // Be sure to call printTokensToString which is verified in unit test.
+  static void printTokens(final Configuration conf,
+      final Path tokenFile,
+      final boolean verbose) throws IOException {
+    System.out.print(printTokensToString(conf, tokenFile, verbose));
   }
 
   private static void printUsage(PrintStream err) {
@@ -216,7 +233,8 @@ public class DelegationTokenFetcher {
     err.println("  --renew             Renew the delegation token.  " +
             "Delegation " + "token must have been fetched using the --renewer" +
             " <name> option.");
-    err.println("  --print             Print the delegation token");
+    err.println("  --print [--verbose] Print the delegation token, when " +
+            "--verbose is passed, print more information about the token");
     err.println();
     GenericOptionsParser.printGenericCommandUsage(err);
     ExitUtil.terminate(1);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java

@@ -322,6 +322,6 @@ public class TestDelegationToken {
         "SomeUser"), new Text("JobTracker"), null);
     Assert.assertEquals("HDFS_DELEGATION_TOKEN token 0" +
         " for SomeUser with renewer JobTracker",
-        dtId.toString());
+        dtId.toStringStable());
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java

@@ -48,8 +48,12 @@ import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestDelegationTokenFetcher {
+  private static final Logger LOG = LoggerFactory.getLogger(
+      TestDelegationTokenFetcher.class);
 
   private Configuration conf = new Configuration();
 
@@ -132,6 +136,19 @@ public class TestDelegationTokenFetcher {
       Iterator<Token<?>> itr = creds.getAllTokens().iterator();
       assertTrue("token not exist error", itr.hasNext());
       assertNotNull("Token should be there without renewer", itr.next());
+
+      // Test compatibility of DelegationTokenFetcher.printTokensToString
+      String expectedNonVerbose = "Token (HDFS_DELEGATION_TOKEN token 1 for " +
+          System.getProperty("user.name") + " with renewer ) for";
+      String resNonVerbose =
+          DelegationTokenFetcher.printTokensToString(conf, p, false);
+      assertTrue("The non verbose output is expected to start with \""
+          + expectedNonVerbose +"\"",
+          resNonVerbose.startsWith(expectedNonVerbose));
+      LOG.info(resNonVerbose);
+      LOG.info(
+          DelegationTokenFetcher.printTokensToString(conf, p, true));
+
       try {
         // Without renewer renewal of token should fail.
         DelegationTokenFetcher.renewTokens(conf, p);