|
|
@@ -18,6 +18,7 @@
|
|
|
|
|
|
package org.apache.hadoop.hdfs.web;
|
|
|
|
|
|
+import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
|
|
|
import static org.junit.Assert.assertEquals;
|
|
|
import static org.junit.Assert.assertNotNull;
|
|
|
import static org.junit.Assert.assertNull;
|
|
|
@@ -26,78 +27,273 @@ import static org.mockito.Mockito.mock;
|
|
|
import java.io.IOException;
|
|
|
import java.net.URI;
|
|
|
import java.net.URL;
|
|
|
+import java.util.Arrays;
|
|
|
|
|
|
import org.apache.hadoop.conf.Configuration;
|
|
|
-import org.apache.hadoop.fs.CommonConfigurationKeys;
|
|
|
import org.apache.hadoop.fs.FileSystem;
|
|
|
import org.apache.hadoop.fs.Path;
|
|
|
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
|
|
|
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
|
|
|
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
|
|
|
-import org.apache.hadoop.hdfs.web.resources.DelegationParam;
|
|
|
-import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
|
|
|
-import org.apache.hadoop.hdfs.web.resources.PutOpParam;
|
|
|
-import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
|
|
|
+import org.apache.hadoop.hdfs.web.resources.*;
|
|
|
import org.apache.hadoop.io.Text;
|
|
|
+import org.apache.hadoop.net.NetUtils;
|
|
|
+import org.apache.hadoop.security.SecurityUtil;
|
|
|
import org.apache.hadoop.security.SecurityUtilTestHelper;
|
|
|
import org.apache.hadoop.security.UserGroupInformation;
|
|
|
import org.apache.hadoop.security.token.Token;
|
|
|
import org.apache.hadoop.security.token.TokenIdentifier;
|
|
|
-import org.junit.Assert;
|
|
|
-import org.junit.Test;
|
|
|
+import org.junit.*;
|
|
|
|
|
|
public class TestWebHdfsUrl {
|
|
|
+ // NOTE: port is never used
|
|
|
+ final URI uri = URI.create(WebHdfsFileSystem.SCHEME + "://" + "127.0.0.1:0");
|
|
|
|
|
|
- @Test
|
|
|
- public void testDelegationTokenInUrl() throws IOException {
|
|
|
+ @Before
|
|
|
+ public void resetUGI() {
|
|
|
+ UserGroupInformation.setConfiguration(new Configuration());
|
|
|
+ }
|
|
|
+
|
|
|
+ @Test(timeout=4000)
|
|
|
+ public void testSimpleAuthParamsInUrl() throws IOException {
|
|
|
+ Configuration conf = new Configuration();
|
|
|
+
|
|
|
+ UserGroupInformation ugi =
|
|
|
+ UserGroupInformation.createRemoteUser("test-user");
|
|
|
+ UserGroupInformation.setLoginUser(ugi);
|
|
|
+
|
|
|
+ WebHdfsFileSystem webhdfs = getWebHdfsFileSystem(ugi, conf);
|
|
|
+ Path fsPath = new Path("/");
|
|
|
+
|
|
|
+ // send user+token
|
|
|
+ URL fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath);
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ GetOpParam.Op.GETFILESTATUS.toQueryString(),
|
|
|
+ new UserParam(ugi.getShortUserName()).toString()
|
|
|
+ },
|
|
|
+ fileStatusUrl);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Test(timeout=4000)
|
|
|
+ public void testSimpleProxyAuthParamsInUrl() throws IOException {
|
|
|
+ Configuration conf = new Configuration();
|
|
|
+
|
|
|
+ UserGroupInformation ugi =
|
|
|
+ UserGroupInformation.createRemoteUser("test-user");
|
|
|
+ ugi = UserGroupInformation.createProxyUser("test-proxy-user", ugi);
|
|
|
+ UserGroupInformation.setLoginUser(ugi);
|
|
|
+
|
|
|
+ WebHdfsFileSystem webhdfs = getWebHdfsFileSystem(ugi, conf);
|
|
|
+ Path fsPath = new Path("/");
|
|
|
+
|
|
|
+ // send real+effective
|
|
|
+ URL fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath);
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ GetOpParam.Op.GETFILESTATUS.toQueryString(),
|
|
|
+ new UserParam(ugi.getRealUser().getShortUserName()).toString(),
|
|
|
+ new DoAsParam(ugi.getShortUserName()).toString()
|
|
|
+ },
|
|
|
+ fileStatusUrl);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Test(timeout=4000)
|
|
|
+ public void testSecureAuthParamsInUrl() throws IOException {
|
|
|
+ Configuration conf = new Configuration();
|
|
|
+ // fake turning on security so api thinks it should use tokens
|
|
|
+ SecurityUtil.setAuthenticationMethod(KERBEROS, conf);
|
|
|
+ UserGroupInformation.setConfiguration(conf);
|
|
|
+
|
|
|
+ UserGroupInformation ugi =
|
|
|
+ UserGroupInformation.createRemoteUser("test-user");
|
|
|
+ ugi.setAuthenticationMethod(KERBEROS);
|
|
|
+ UserGroupInformation.setLoginUser(ugi);
|
|
|
+
|
|
|
+ WebHdfsFileSystem webhdfs = getWebHdfsFileSystem(ugi, conf);
|
|
|
+ Path fsPath = new Path("/");
|
|
|
+ String tokenString = webhdfs.getRenewToken().encodeToUrlString();
|
|
|
+
|
|
|
+ // send user
|
|
|
+ URL getTokenUrl = webhdfs.toUrl(GetOpParam.Op.GETDELEGATIONTOKEN, fsPath);
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ GetOpParam.Op.GETDELEGATIONTOKEN.toQueryString(),
|
|
|
+ new UserParam(ugi.getShortUserName()).toString()
|
|
|
+ },
|
|
|
+ getTokenUrl);
|
|
|
+
|
|
|
+ // send user
|
|
|
+ URL renewTokenUrl = webhdfs.toUrl(PutOpParam.Op.RENEWDELEGATIONTOKEN,
|
|
|
+ fsPath, new TokenArgumentParam(tokenString));
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ PutOpParam.Op.RENEWDELEGATIONTOKEN.toQueryString(),
|
|
|
+ new UserParam(ugi.getShortUserName()).toString(),
|
|
|
+ new TokenArgumentParam(tokenString).toString(),
|
|
|
+ },
|
|
|
+ renewTokenUrl);
|
|
|
+
|
|
|
+ // send user+token
|
|
|
+ URL cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN,
|
|
|
+ fsPath, new TokenArgumentParam(tokenString));
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ PutOpParam.Op.CANCELDELEGATIONTOKEN.toQueryString(),
|
|
|
+ new UserParam(ugi.getShortUserName()).toString(),
|
|
|
+ new TokenArgumentParam(tokenString).toString(),
|
|
|
+ new DelegationParam(tokenString).toString()
|
|
|
+ },
|
|
|
+ cancelTokenUrl);
|
|
|
+
|
|
|
+ // send user+token
|
|
|
+ URL fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath);
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ GetOpParam.Op.GETFILESTATUS.toQueryString(),
|
|
|
+ new UserParam(ugi.getShortUserName()).toString(),
|
|
|
+ new DelegationParam(tokenString).toString()
|
|
|
+ },
|
|
|
+ fileStatusUrl);
|
|
|
+
|
|
|
+ // wipe out internal token to simulate auth always required
|
|
|
+ webhdfs.setDelegationToken(null);
|
|
|
+
|
|
|
+ // send user
|
|
|
+ cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN,
|
|
|
+ fsPath, new TokenArgumentParam(tokenString));
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ PutOpParam.Op.CANCELDELEGATIONTOKEN.toQueryString(),
|
|
|
+ new UserParam(ugi.getShortUserName()).toString(),
|
|
|
+ new TokenArgumentParam(tokenString).toString(),
|
|
|
+ },
|
|
|
+ cancelTokenUrl);
|
|
|
+
|
|
|
+ // send user
|
|
|
+ fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath);
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ GetOpParam.Op.GETFILESTATUS.toQueryString(),
|
|
|
+ new UserParam(ugi.getShortUserName()).toString()
|
|
|
+ },
|
|
|
+ fileStatusUrl);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Test(timeout=4000)
|
|
|
+ public void testSecureProxyAuthParamsInUrl() throws IOException {
|
|
|
Configuration conf = new Configuration();
|
|
|
- final String uri = WebHdfsFileSystem.SCHEME + "://" + "127.0.0.1:9071";
|
|
|
- // Turn on security
|
|
|
- conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
|
|
|
+ // fake turning on security so api thinks it should use tokens
|
|
|
+ SecurityUtil.setAuthenticationMethod(KERBEROS, conf);
|
|
|
UserGroupInformation.setConfiguration(conf);
|
|
|
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
|
|
- DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(
|
|
|
- ugi.getUserName()), null, null);
|
|
|
- FSNamesystem namesystem = mock(FSNamesystem.class);
|
|
|
- DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager(
|
|
|
- 86400000, 86400000, 86400000, 86400000, namesystem);
|
|
|
- dtSecretManager.startThreads();
|
|
|
- Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
|
|
|
- dtId, dtSecretManager);
|
|
|
- token.setService(new Text("127.0.0.1:9071"));
|
|
|
- token.setKind(WebHdfsFileSystem.TOKEN_KIND);
|
|
|
- ugi.addToken(token);
|
|
|
- final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem) FileSystem.get(
|
|
|
- URI.create(uri), conf);
|
|
|
- String tokenString = token.encodeToUrlString();
|
|
|
+
|
|
|
+ UserGroupInformation ugi =
|
|
|
+ UserGroupInformation.createRemoteUser("test-user");
|
|
|
+ ugi.setAuthenticationMethod(KERBEROS);
|
|
|
+ ugi = UserGroupInformation.createProxyUser("test-proxy-user", ugi);
|
|
|
+ UserGroupInformation.setLoginUser(ugi);
|
|
|
+
|
|
|
+ WebHdfsFileSystem webhdfs = getWebHdfsFileSystem(ugi, conf);
|
|
|
Path fsPath = new Path("/");
|
|
|
+ String tokenString = webhdfs.getRenewToken().encodeToUrlString();
|
|
|
+
|
|
|
+ // send real+effective
|
|
|
+ URL getTokenUrl = webhdfs.toUrl(GetOpParam.Op.GETDELEGATIONTOKEN, fsPath);
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ GetOpParam.Op.GETDELEGATIONTOKEN.toQueryString(),
|
|
|
+ new UserParam(ugi.getRealUser().getShortUserName()).toString(),
|
|
|
+ new DoAsParam(ugi.getShortUserName()).toString()
|
|
|
+ },
|
|
|
+ getTokenUrl);
|
|
|
+
|
|
|
+ // send real+effective
|
|
|
URL renewTokenUrl = webhdfs.toUrl(PutOpParam.Op.RENEWDELEGATIONTOKEN,
|
|
|
fsPath, new TokenArgumentParam(tokenString));
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ PutOpParam.Op.RENEWDELEGATIONTOKEN.toQueryString(),
|
|
|
+ new UserParam(ugi.getRealUser().getShortUserName()).toString(),
|
|
|
+ new DoAsParam(ugi.getShortUserName()).toString(),
|
|
|
+ new TokenArgumentParam(tokenString).toString(),
|
|
|
+ },
|
|
|
+ renewTokenUrl);
|
|
|
+
|
|
|
+ // send effective+token
|
|
|
URL cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN,
|
|
|
fsPath, new TokenArgumentParam(tokenString));
|
|
|
- Assert.assertEquals(
|
|
|
- generateUrlQueryPrefix(PutOpParam.Op.RENEWDELEGATIONTOKEN,
|
|
|
- ugi.getUserName())
|
|
|
- + "&token=" + tokenString, renewTokenUrl.getQuery());
|
|
|
- Token<DelegationTokenIdentifier> delegationToken = new Token<DelegationTokenIdentifier>(
|
|
|
- token);
|
|
|
- delegationToken.setKind(WebHdfsFileSystem.TOKEN_KIND);
|
|
|
- Assert.assertEquals(
|
|
|
- generateUrlQueryPrefix(PutOpParam.Op.CANCELDELEGATIONTOKEN,
|
|
|
- ugi.getUserName())
|
|
|
- + "&token="
|
|
|
- + tokenString
|
|
|
- + "&"
|
|
|
- + DelegationParam.NAME
|
|
|
- + "="
|
|
|
- + delegationToken.encodeToUrlString(), cancelTokenUrl.getQuery());
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ PutOpParam.Op.CANCELDELEGATIONTOKEN.toQueryString(),
|
|
|
+ new UserParam(ugi.getShortUserName()).toString(),
|
|
|
+ new TokenArgumentParam(tokenString).toString(),
|
|
|
+ new DelegationParam(tokenString).toString()
|
|
|
+ },
|
|
|
+ cancelTokenUrl);
|
|
|
+
|
|
|
+ // send effective+token
|
|
|
+ URL fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath);
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ GetOpParam.Op.GETFILESTATUS.toQueryString(),
|
|
|
+ new UserParam(ugi.getShortUserName()).toString(),
|
|
|
+ new DelegationParam(tokenString).toString()
|
|
|
+ },
|
|
|
+ fileStatusUrl);
|
|
|
+
|
|
|
+ // wipe out internal token to simulate auth always required
|
|
|
+ webhdfs.setDelegationToken(null);
|
|
|
+
|
|
|
+ // send real+effective
|
|
|
+ cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN,
|
|
|
+ fsPath, new TokenArgumentParam(tokenString));
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ PutOpParam.Op.CANCELDELEGATIONTOKEN.toQueryString(),
|
|
|
+ new UserParam(ugi.getRealUser().getShortUserName()).toString(),
|
|
|
+ new DoAsParam(ugi.getShortUserName()).toString(),
|
|
|
+ new TokenArgumentParam(tokenString).toString()
|
|
|
+ },
|
|
|
+ cancelTokenUrl);
|
|
|
+
|
|
|
+ // send real+effective
|
|
|
+ fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath);
|
|
|
+ checkQueryParams(
|
|
|
+ new String[]{
|
|
|
+ GetOpParam.Op.GETFILESTATUS.toQueryString(),
|
|
|
+ new UserParam(ugi.getRealUser().getShortUserName()).toString(),
|
|
|
+ new DoAsParam(ugi.getShortUserName()).toString()
|
|
|
+ },
|
|
|
+ fileStatusUrl);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void checkQueryParams(String[] expected, URL url) {
|
|
|
+ Arrays.sort(expected);
|
|
|
+ String[] query = url.getQuery().split("&");
|
|
|
+ Arrays.sort(query);
|
|
|
+ assertEquals(Arrays.toString(expected), Arrays.toString(query));
|
|
|
}
|
|
|
|
|
|
- private String generateUrlQueryPrefix(HttpOpParam.Op op, String username) {
|
|
|
- return "op=" + op.toString() + "&user.name=" + username;
|
|
|
+ private WebHdfsFileSystem getWebHdfsFileSystem(UserGroupInformation ugi,
|
|
|
+ Configuration conf) throws IOException {
|
|
|
+ if (UserGroupInformation.isSecurityEnabled()) {
|
|
|
+ DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(
|
|
|
+ ugi.getUserName()), null, null);
|
|
|
+ FSNamesystem namesystem = mock(FSNamesystem.class);
|
|
|
+ DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager(
|
|
|
+ 86400000, 86400000, 86400000, 86400000, namesystem);
|
|
|
+ dtSecretManager.startThreads();
|
|
|
+ Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
|
|
|
+ dtId, dtSecretManager);
|
|
|
+ SecurityUtil.setTokenService(
|
|
|
+ token, NetUtils.createSocketAddr(uri.getAuthority()));
|
|
|
+ token.setKind(WebHdfsFileSystem.TOKEN_KIND);
|
|
|
+ ugi.addToken(token);
|
|
|
+ }
|
|
|
+ return (WebHdfsFileSystem) FileSystem.get(uri, conf);
|
|
|
}
|
|
|
|
|
|
- @Test
|
|
|
+ @Test(timeout=4000)
|
|
|
public void testSelectHdfsDelegationToken() throws Exception {
|
|
|
SecurityUtilTestHelper.setTokenServiceUseIp(true);
|
|
|
|
Suresh Srinivas