HADOOP-15885. Add base64 (urlString) support to DTUtil. Contributed by Inigo Goiri.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java

@@ -89,7 +89,7 @@ public final class DtFileOperations {
 
   /** Add the service prefix for a local filesystem. */
   private static Path fileToPath(File f) {
-    return new Path("file:" + f.getAbsolutePath());
+    return new Path(f.toURI().toString());
   }
 
   /** Write out a Credentials object as a local file.
@@ -294,4 +294,30 @@ public final class DtFileOperations {
     }
     doFormattedWrite(tokenFile, fileFormat, creds, conf);
   }
+
+  /** Import a token from a base64 encoding into the local filesystem.
+   * @param tokenFile A local File object.
+   * @param fileFormat A string equal to FORMAT_PB or FORMAT_JAVA, for output.
+   * @param alias overwrite Service field of fetched token with this text.
+   * @param base64 urlString Encoding of the token to import.
+   * @param conf Configuration object passed along.
+   * @throws IOException Error to import the token into the file.
+   */
+  public static void importTokenFile(File tokenFile, String fileFormat,
+      Text alias, String base64, Configuration conf)
+      throws IOException {
+
+    Credentials creds = tokenFile.exists() ?
+        Credentials.readTokenStorageFile(tokenFile, conf) : new Credentials();
+
+    Token<TokenIdentifier> token = new Token<>();
+    token.decodeFromUrlString(base64);
+    if (alias != null) {
+      token.setService(alias);
+    }
+    creds.addToken(token.getService(), token);
+    LOG.info("Add token with service {}", token.getService());
+
+    doFormattedWrite(tokenFile, fileFormat, creds, conf);
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java

@@ -55,6 +55,7 @@ public class DtUtilShell extends CommandShell {
   private static final String CANCEL = "cancel";
   private static final String REMOVE = "remove";
   private static final String RENEW = "renew";
+  private static final String IMPORT = "import";
   private static final String RENEWER = "-renewer";
   private static final String SERVICE = "-service";
   private static final String ALIAS = "-alias";
@@ -138,6 +139,8 @@ public class DtUtilShell extends CommandShell {
           setSubCommand(new Remove(false));
         } else if (command.equals(RENEW)) {
           setSubCommand(new Renew());
+        } else if (command.equals(IMPORT)) {
+          setSubCommand(new Import(args[++i]));
         }
       } else if (args[i].equals(ALIAS)) {
         alias = new Text(args[++i]);
@@ -176,11 +179,11 @@ public class DtUtilShell extends CommandShell {
   @Override
   public String getCommandUsage() {
     return String.format(
-        "%n%s%n   %s%n   %s%n   %s%n   %s%n   %s%n   %s%n   %s%n%n",
+        "%n%s%n   %s%n   %s%n   %s%n   %s%n   %s%n   %s%n   %s%n   %s%n%n",
         DT_USAGE, (new Print()).getUsage(), (new Get()).getUsage(),
         (new Edit()).getUsage(), (new Append()).getUsage(),
         (new Remove(true)).getUsage(), (new Remove(false)).getUsage(),
-        (new Renew()).getUsage());
+        (new Renew()).getUsage(), (new Import()).getUsage());
   }
 
   private class Print extends SubCommand {
@@ -357,6 +360,36 @@ public class DtUtilShell extends CommandShell {
     }
   }
 
+  private class Import extends SubCommand {
+    public static final String IMPORT_USAGE =
+        "dtutil import <base64> [-alias <alias>] " +
+        FORMAT_SUBSTRING + " filename";
+
+    private String base64 = null;
+
+    Import() { }
+
+    Import(String arg) {
+      base64 = arg;
+    }
+
+    @Override
+    public boolean validate() {
+      return true;
+    }
+
+    @Override
+    public void execute() throws Exception {
+      DtFileOperations.importTokenFile(
+          firstFile, format, alias, base64, getConf());
+    }
+
+    @Override
+    public String getUsage() {
+      return IMPORT_USAGE;
+    }
+  }
+
   public static void main(String[] args) throws Exception {
     System.exit(ToolRunner.run(new Configuration(), new DtUtilShell(), args));
   }

hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md

@@ -172,6 +172,7 @@ For every subcommand that connects to a service, convenience flags are provided
 | `remove -alias` *alias* <br/>&nbsp;&nbsp; `[-format (java|protobuf)]` <br/>&nbsp;&nbsp; *filename* `[` *filename2* `...]` | From each file specified, remove the tokens matching *alias* and write out each file using specified format. <br/>  *alias* must be specified. |
 | `cancel -alias` *alias* <br/>&nbsp;&nbsp; `[-format (java|protobuf)]` <br/>&nbsp;&nbsp;  *filename* `[` *filename2* `...]` | Just like `remove`, except the tokens are also cancelled using the service specified in the token object. <br/> *alias* must be specified. |
 | `renew -alias` *alias* <br/>&nbsp;&nbsp; `[-format (java|protobuf)]` <br/>&nbsp;&nbsp;  *filename* `[` *filename2* `...]` | For each file specified, renew the tokens matching *alias* and write out each file using specified format. <br/> *alias* must be specified. |
+| `import` *base64* <br/>&nbsp;&nbsp;  `[-alias` *alias* `]` <br/>&nbsp;&nbsp; *filename* | Import a token from a base64 token. <br/> *alias* will overwrite the service field in the token. |
 
 ### `fs`
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java

@@ -57,6 +57,13 @@ public class TestDtUtilShell {
   public static Text KIND_GET = new Text("testTokenKindGet");
   public static Token<?> MOCK_TOKEN =
       new Token(IDENTIFIER, PASSWORD, KIND_GET, SERVICE_GET);
+
+  private static final Text SERVICE_IMPORT =
+      new Text("testTokenServiceImport");
+  private static final Text KIND_IMPORT = new Text("testTokenKindImport");
+  private static final Token<?> IMPORT_TOKEN =
+      new Token(IDENTIFIER, PASSWORD, KIND_IMPORT, SERVICE_IMPORT);
+
   static {
     try {
       defaultConf.set("fs.defaultFS", "file:///");
@@ -73,9 +80,11 @@ public class TestDtUtilShell {
   private final Path tokenFile2 = new Path(workDir, "testPrintTokenFile2");
   private final Path tokenLegacyFile = new Path(workDir, "testPrintTokenFile3");
   private final Path tokenFileGet = new Path(workDir, "testGetTokenFile");
+  private final Path tokenFileImport = new Path(workDir, "testImportTokenFile");
   private final String tokenFilename = tokenFile.toString();
   private final String tokenFilename2 = tokenFile2.toString();
   private final String tokenFilenameGet = tokenFileGet.toString();
+  private final String tokenFilenameImport = tokenFileImport.toString();
   private String[] args = null;
   private DtUtilShell dt = null;
   private int rc = 0;
@@ -283,4 +292,39 @@ public class TestDtUtilShell {
     spyCreds.readTokenStorageStream(in);
     Mockito.verify(spyCreds, Mockito.never()).readFields(in);
   }
+
+  @Test
+  public void testImport() throws Exception {
+    String base64 = IMPORT_TOKEN.encodeToUrlString();
+    args = new String[] {"import", base64, tokenFilenameImport};
+    rc = dt.run(args);
+    assertEquals("test simple import print old exit code", 0, rc);
+
+    args = new String[] {"print", tokenFilenameImport};
+    rc = dt.run(args);
+    assertEquals("test simple import print old exit code", 0, rc);
+    assertTrue("test print after import output:\n" + outContent,
+               outContent.toString().contains(KIND_IMPORT.toString()));
+    assertTrue("test print after import output:\n" + outContent,
+        outContent.toString().contains(SERVICE_IMPORT.toString()));
+    assertTrue("test print after simple import output:\n" + outContent,
+               outContent.toString().contains(base64));
+  }
+
+  @Test
+  public void testImportWithAliasFlag() throws Exception {
+    String base64 = IMPORT_TOKEN.encodeToUrlString();
+    args = new String[] {"import", base64, "-alias", alias,
+        tokenFilenameImport};
+    rc = dt.run(args);
+    assertEquals("test import with alias print old exit code", 0, rc);
+
+    args = new String[] {"print", tokenFilenameImport};
+    rc = dt.run(args);
+    assertEquals("test simple import print old exit code", 0, rc);
+    assertTrue("test print after import output:\n" + outContent,
+               outContent.toString().contains(KIND_IMPORT.toString()));
+    assertTrue("test print after import with alias output:\n" + outContent,
+               outContent.toString().contains(alias));
+  }
 }