|
|
@@ -1,1940 +0,0 @@
|
|
|
-/**
|
|
|
- * Licensed to the Apache Software Foundation (ASF) under one
|
|
|
- * or more contributor license agreements. See the NOTICE file
|
|
|
- * distributed with this work for additional information
|
|
|
- * regarding copyright ownership. The ASF licenses this file
|
|
|
- * to you under the Apache License, Version 2.0 (the
|
|
|
- * "License"); you may not use this file except in compliance
|
|
|
- * with the License. You may obtain a copy of the License at
|
|
|
- *
|
|
|
- * http://www.apache.org/licenses/LICENSE-2.0
|
|
|
- *
|
|
|
- * Unless required by applicable law or agreed to in writing, software
|
|
|
- * distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
- * See the License for the specific language governing permissions and
|
|
|
- * limitations under the License.
|
|
|
- */
|
|
|
-package org.apache.hadoop.tools.rumen;
|
|
|
-
|
|
|
-import java.io.DataInput;
|
|
|
-import java.io.DataOutput;
|
|
|
-import java.io.File;
|
|
|
-import java.io.IOException;
|
|
|
-import java.io.OutputStream;
|
|
|
-import java.util.List;
|
|
|
-import java.util.Map;
|
|
|
-import java.util.Properties;
|
|
|
-import java.util.concurrent.TimeUnit;
|
|
|
-
|
|
|
-import org.apache.hadoop.conf.Configuration;
|
|
|
-import org.apache.hadoop.fs.FSDataInputStream;
|
|
|
-import org.apache.hadoop.fs.FSDataOutputStream;
|
|
|
-import org.apache.hadoop.fs.FileSystem;
|
|
|
-import org.apache.hadoop.fs.Path;
|
|
|
-import org.apache.hadoop.fs.permission.FsPermission;
|
|
|
-import org.apache.hadoop.hdfs.MiniDFSCluster;
|
|
|
-import org.apache.hadoop.io.WritableUtils;
|
|
|
-import org.apache.hadoop.mapred.JobConf;
|
|
|
-import org.apache.hadoop.mapred.MiniMRCluster;
|
|
|
-import org.apache.hadoop.mapreduce.Cluster;
|
|
|
-import org.apache.hadoop.mapreduce.ID;
|
|
|
-import org.apache.hadoop.mapreduce.Job;
|
|
|
-import org.apache.hadoop.mapreduce.JobID;
|
|
|
-import org.apache.hadoop.mapreduce.MRJobConfig;
|
|
|
-import org.apache.hadoop.mapreduce.MapReduceTestUtil;
|
|
|
-import org.apache.hadoop.mapreduce.TaskAttemptID;
|
|
|
-import org.apache.hadoop.mapreduce.TaskID;
|
|
|
-import org.apache.hadoop.mapreduce.TaskType;
|
|
|
-import org.apache.hadoop.mapreduce.jobhistory.JobHistory;
|
|
|
-import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
|
|
|
-import org.apache.hadoop.security.UserGroupInformation;
|
|
|
-import org.apache.hadoop.tools.rumen.anonymization.*;
|
|
|
-import org.apache.hadoop.tools.rumen.datatypes.*;
|
|
|
-import org.apache.hadoop.tools.rumen.datatypes.FileName.FileNameState;
|
|
|
-import org.apache.hadoop.tools.rumen.datatypes.NodeName.NodeNameState;
|
|
|
-import org.apache.hadoop.tools.rumen.datatypes.util.DefaultJobPropertiesParser;
|
|
|
-import org.apache.hadoop.tools.rumen.datatypes.util.MapReduceJobPropertiesParser;
|
|
|
-import org.apache.hadoop.tools.rumen.serializers.*;
|
|
|
-import org.apache.hadoop.tools.rumen.state.*;
|
|
|
-import org.apache.hadoop.util.Tool;
|
|
|
-import org.apache.hadoop.util.ToolRunner;
|
|
|
-import org.codehaus.jackson.JsonEncoding;
|
|
|
-import org.codehaus.jackson.JsonFactory;
|
|
|
-import org.codehaus.jackson.JsonGenerator;
|
|
|
-import org.codehaus.jackson.Version;
|
|
|
-import org.codehaus.jackson.annotate.JsonIgnore;
|
|
|
-import org.codehaus.jackson.map.JsonSerializer;
|
|
|
-import org.codehaus.jackson.map.ObjectMapper;
|
|
|
-import org.codehaus.jackson.map.module.SimpleModule;
|
|
|
-import org.junit.Test;
|
|
|
-import static org.junit.Assert.*;
|
|
|
-
|
|
|
-/**
|
|
|
- * Tests Rumen output anonymization.
|
|
|
- */
|
|
|
-@SuppressWarnings("deprecation")
|
|
|
-public class TestRumenAnonymization {
|
|
|
- /**
|
|
|
- * Test {@link UserName}, serialization and anonymization.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testUserNameSerialization() throws IOException {
|
|
|
- JsonSerializer<?> defaultSerializer = new DefaultRumenSerializer();
|
|
|
-
|
|
|
- JsonSerializer<?> anonymizingSerializer =
|
|
|
- new DefaultAnonymizingRumenSerializer(new StatePool(),
|
|
|
- new Configuration());
|
|
|
- // test username
|
|
|
- UserName uname = new UserName("bob");
|
|
|
- assertEquals("Username error!", "bob", uname.getValue());
|
|
|
-
|
|
|
- // test username serialization
|
|
|
- // test with no anonymization
|
|
|
- // test bob
|
|
|
- testSerializer(new UserName("bob"), "bob", defaultSerializer);
|
|
|
- // test alice
|
|
|
- testSerializer(new UserName("alice"), "alice", defaultSerializer);
|
|
|
-
|
|
|
- // test user-name serialization
|
|
|
- // test with anonymization
|
|
|
- // test bob
|
|
|
- testSerializer(new UserName("bob"), "user0", anonymizingSerializer);
|
|
|
- // test alice
|
|
|
- testSerializer(new UserName("alice"), "user1", anonymizingSerializer);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link JobName}, serialization and anonymization.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testJobNameSerialization() throws IOException {
|
|
|
- JsonSerializer<?> defaultSerializer = new DefaultRumenSerializer();
|
|
|
-
|
|
|
- JsonSerializer<?> anonymizingSerializer =
|
|
|
- new DefaultAnonymizingRumenSerializer(new StatePool(),
|
|
|
- new Configuration());
|
|
|
-
|
|
|
- // test jobname
|
|
|
- JobName jname = new JobName("job-secret");
|
|
|
- assertEquals("Jobname error!", "job-secret", jname.getValue());
|
|
|
-
|
|
|
- // test job-name serialization
|
|
|
- // test with no anonymization
|
|
|
- // test job1
|
|
|
- testSerializer(new JobName("job-myjob"), "job-myjob", defaultSerializer);
|
|
|
- // test job2
|
|
|
- testSerializer(new JobName("job-yourjob"), "job-yourjob",
|
|
|
- defaultSerializer);
|
|
|
-
|
|
|
- // test job-name serialization
|
|
|
- // test with anonymization
|
|
|
- // test queue1
|
|
|
- testSerializer(new JobName("secret-job-1"), "job0", anonymizingSerializer);
|
|
|
- // test queue2
|
|
|
- testSerializer(new JobName("secret-job-2"), "job1", anonymizingSerializer);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link QueueName}, serialization and anonymization.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testQueueNameSerialization() throws IOException {
|
|
|
- JsonSerializer<?> defaultSerializer = new DefaultRumenSerializer();
|
|
|
-
|
|
|
- JsonSerializer<?> anonymizingSerializer =
|
|
|
- new DefaultAnonymizingRumenSerializer(new StatePool(),
|
|
|
- new Configuration());
|
|
|
-
|
|
|
- // test queuename
|
|
|
- QueueName qname = new QueueName("queue-secret");
|
|
|
- assertEquals("Queuename error!", "queue-secret", qname.getValue());
|
|
|
-
|
|
|
- // test queuename serialization
|
|
|
- // test with no anonymization
|
|
|
- // test queue1
|
|
|
- testSerializer(new QueueName("project1-queue"),
|
|
|
- "project1-queue", defaultSerializer);
|
|
|
- // test queue2
|
|
|
- testSerializer(new QueueName("project2-queue"),
|
|
|
- "project2-queue", defaultSerializer);
|
|
|
-
|
|
|
- // test queue-name serialization
|
|
|
- // test with anonymization
|
|
|
- // test queue1
|
|
|
- testSerializer(new QueueName("project1-queue"),
|
|
|
- "queue0", anonymizingSerializer);
|
|
|
- // test queue2
|
|
|
- testSerializer(new QueueName("project2-queue"),
|
|
|
- "queue1", anonymizingSerializer);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link NodeName}.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testNodeNameDataType() throws IOException {
|
|
|
- // test hostname
|
|
|
- // test only hostname
|
|
|
- NodeName hname = new NodeName("host1.myorg.com");
|
|
|
- assertNull("Expected missing rack name", hname.getRackName());
|
|
|
- assertEquals("Hostname's test#1 hostname error!",
|
|
|
- "host1.myorg.com", hname.getHostName());
|
|
|
- assertEquals("Hostname test#1 error!", "host1.myorg.com", hname.getValue());
|
|
|
-
|
|
|
- // test rack/hostname
|
|
|
- hname = new NodeName("/rack1.myorg.com/host1.myorg.com");
|
|
|
- assertEquals("Hostname's test#2 rackname error!",
|
|
|
- "rack1.myorg.com", hname.getRackName());
|
|
|
- assertEquals("Hostname test#2 hostname error!",
|
|
|
- "host1.myorg.com", hname.getHostName());
|
|
|
- assertEquals("Hostname test#2 error!",
|
|
|
- "/rack1.myorg.com/host1.myorg.com", hname.getValue());
|
|
|
-
|
|
|
- // test hostname and rackname separately
|
|
|
- hname = new NodeName("rack1.myorg.com", "host1.myorg.com");
|
|
|
- assertEquals("Hostname's test#3 rackname error!",
|
|
|
- "rack1.myorg.com", hname.getRackName());
|
|
|
- assertEquals("Hostname test#3 hostname error!",
|
|
|
- "host1.myorg.com", hname.getHostName());
|
|
|
- assertEquals("Hostname test#3 error!",
|
|
|
- "/rack1.myorg.com/host1.myorg.com", hname.getValue());
|
|
|
-
|
|
|
- // test hostname with no rackname
|
|
|
- hname = new NodeName(null, "host1.myorg.com");
|
|
|
- assertNull("Hostname's test#4 rackname error!", hname.getRackName());
|
|
|
- assertEquals("Hostname test#4 hostname error!",
|
|
|
- "host1.myorg.com", hname.getHostName());
|
|
|
- assertEquals("Hostname test#4 error!",
|
|
|
- "host1.myorg.com", hname.getValue());
|
|
|
-
|
|
|
- // test rackname with no hostname
|
|
|
- hname = new NodeName("rack1.myorg.com", null);
|
|
|
- assertEquals("Hostname test#4 rackname error!",
|
|
|
- "rack1.myorg.com", hname.getRackName());
|
|
|
- assertNull("Hostname's test#5 hostname error!", hname.getHostName());
|
|
|
- assertEquals("Hostname test#5 error!",
|
|
|
- "rack1.myorg.com", hname.getValue());
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link NodeName} serialization.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testNodeNameDefaultSerialization() throws IOException {
|
|
|
- JsonSerializer<?> defaultSerializer = new DefaultRumenSerializer();
|
|
|
-
|
|
|
- // test hostname serialization
|
|
|
- // test with no anonymization
|
|
|
- // test hostname
|
|
|
- testSerializer(new NodeName("hostname.myorg.com"), "hostname.myorg.com",
|
|
|
- defaultSerializer);
|
|
|
- // test rack/hostname
|
|
|
- testSerializer(new NodeName("/rackname.myorg.com/hostname.myorg.com"),
|
|
|
- "/rackname.myorg.com/hostname.myorg.com",
|
|
|
- defaultSerializer);
|
|
|
- // test rack,hostname
|
|
|
- testSerializer(new NodeName("rackname.myorg.com", "hostname.myorg.com"),
|
|
|
- "/rackname.myorg.com/hostname.myorg.com",
|
|
|
- defaultSerializer);
|
|
|
- // test -,hostname
|
|
|
- testSerializer(new NodeName(null, "hostname.myorg.com"),
|
|
|
- "hostname.myorg.com", defaultSerializer);
|
|
|
- // test rack,-
|
|
|
- testSerializer(new NodeName("rackname.myorg.com", null),
|
|
|
- "rackname.myorg.com", defaultSerializer);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link NodeName} anonymization.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testNodeNameAnonymization() throws IOException {
|
|
|
- JsonSerializer<?> anonymizingSerializer =
|
|
|
- new DefaultAnonymizingRumenSerializer(new StatePool(),
|
|
|
- new Configuration());
|
|
|
-
|
|
|
- // test hostname serializer
|
|
|
- // test with anonymization
|
|
|
- // test hostname
|
|
|
- testSerializer(new NodeName("hostname.myorg.com"), "host0",
|
|
|
- anonymizingSerializer);
|
|
|
- // test hostname reuse
|
|
|
- testSerializer(new NodeName("hostname213.myorg.com"), "host1",
|
|
|
- anonymizingSerializer);
|
|
|
- // test rack/hostname
|
|
|
- testSerializer(new NodeName("/rackname.myorg.com/hostname.myorg.com"),
|
|
|
- "/rack0/host0", anonymizingSerializer);
|
|
|
- // test rack/hostname (hostname reuse)
|
|
|
- testSerializer(new NodeName("/rackname654.myorg.com/hostname.myorg.com"),
|
|
|
- "/rack1/host0", anonymizingSerializer);
|
|
|
- // test rack/hostname (rack reuse)
|
|
|
- testSerializer(new NodeName("/rackname654.myorg.com/hostname765.myorg.com"),
|
|
|
- "/rack1/host2", anonymizingSerializer);
|
|
|
- // test rack,hostname (rack & hostname reuse)
|
|
|
- testSerializer(new NodeName("rackname.myorg.com", "hostname.myorg.com"),
|
|
|
- "/rack0/host0", anonymizingSerializer);
|
|
|
- // test rack,hostname (rack reuse)
|
|
|
- testSerializer(new NodeName("rackname.myorg.com", "hostname543.myorg.com"),
|
|
|
- "/rack0/host3", anonymizingSerializer);
|
|
|
- // test rack,hostname (hostname reuse)
|
|
|
- testSerializer(new NodeName("rackname987.myorg.com", "hostname.myorg.com"),
|
|
|
- "/rack2/host0", anonymizingSerializer);
|
|
|
- // test rack,hostname (rack reuse)
|
|
|
- testSerializer(new NodeName("rackname.myorg.com", "hostname654.myorg.com"),
|
|
|
- "/rack0/host4", anonymizingSerializer);
|
|
|
- // test rack,hostname (host reuse)
|
|
|
- testSerializer(new NodeName("rackname876.myorg.com", "hostname.myorg.com"),
|
|
|
- "/rack3/host0", anonymizingSerializer);
|
|
|
- // test rack,hostname (rack & hostname reuse)
|
|
|
- testSerializer(new NodeName("rackname987.myorg.com",
|
|
|
- "hostname543.myorg.com"),
|
|
|
- "/rack2/host3", anonymizingSerializer);
|
|
|
- // test -,hostname (hostname reuse)
|
|
|
- testSerializer(new NodeName(null, "hostname.myorg.com"),
|
|
|
- "host0", anonymizingSerializer);
|
|
|
- // test -,hostname
|
|
|
- testSerializer(new NodeName(null, "hostname15.myorg.com"),
|
|
|
- "host5", anonymizingSerializer);
|
|
|
- // test rack,- (rack reuse)
|
|
|
- testSerializer(new NodeName("rackname987.myorg.com", null),
|
|
|
- "rack2", anonymizingSerializer);
|
|
|
- // test rack,-
|
|
|
- testSerializer(new NodeName("rackname15.myorg.com", null),
|
|
|
- "rack4", anonymizingSerializer);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link JobProperties}.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testJobPropertiesDataType() throws IOException {
|
|
|
- // test job properties
|
|
|
- Properties properties = new Properties();
|
|
|
- JobProperties jp = new JobProperties(properties);
|
|
|
-
|
|
|
- // test empty job properties
|
|
|
- assertEquals("Job Properties (default) store error",
|
|
|
- 0, jp.getValue().size());
|
|
|
- // test by adding some data
|
|
|
- properties.put("test-key", "test-value"); // user config
|
|
|
- properties.put(MRJobConfig.USER_NAME, "bob"); // job config
|
|
|
- properties.put(JobConf.MAPRED_TASK_JAVA_OPTS, "-Xmx1G"); // deprecated
|
|
|
- jp = new JobProperties(properties);
|
|
|
- assertEquals("Job Properties (default) store error",
|
|
|
- 3, jp.getValue().size());
|
|
|
- assertEquals("Job Properties (default) key#1 error",
|
|
|
- "test-value", jp.getValue().get("test-key"));
|
|
|
- assertEquals("Job Properties (default) key#2 error",
|
|
|
- "bob", jp.getValue().get(MRJobConfig.USER_NAME));
|
|
|
- assertEquals("Job Properties (default) key#3 error",
|
|
|
- "-Xmx1G", jp.getValue().get(JobConf.MAPRED_TASK_JAVA_OPTS));
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link JobProperties} serialization.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testJobPropertiesSerialization() throws IOException {
|
|
|
- JsonSerializer<?> defaultSerializer = new DefaultRumenSerializer();
|
|
|
-
|
|
|
- // test job properties
|
|
|
- Properties properties = new Properties();
|
|
|
- properties.put("test-key", "test-value"); // user config
|
|
|
- properties.put(MRJobConfig.USER_NAME, "bob"); // job config
|
|
|
- properties.put(JobConf.MAPRED_TASK_JAVA_OPTS, "-Xmx1G"); // deprecated
|
|
|
- JobProperties jp = new JobProperties(properties);
|
|
|
-
|
|
|
- testSerializer(jp, "{test-key:test-value,"
|
|
|
- + "mapreduce.job.user.name:bob,"
|
|
|
- + "mapred.child.java.opts:-Xmx1G}", defaultSerializer);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link JobProperties} anonymization.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testJobPropertiesAnonymization() throws IOException {
|
|
|
- // test job properties
|
|
|
- Properties properties = new Properties();
|
|
|
- Configuration conf = new Configuration();
|
|
|
-
|
|
|
- properties.put("test-key", "test-value"); // user config
|
|
|
- properties.put(MRJobConfig.USER_NAME, "bob"); // job config
|
|
|
- // deprecated
|
|
|
- properties.put("mapred.map.child.java.opts",
|
|
|
- "-Xmx2G -Xms500m -Dsecret=secret");
|
|
|
- // deprecated and not supported
|
|
|
- properties.put(JobConf.MAPRED_TASK_JAVA_OPTS,
|
|
|
- "-Xmx1G -Xms200m -Dsecret=secret");
|
|
|
- JobProperties jp = new JobProperties(properties);
|
|
|
-
|
|
|
- // define a module
|
|
|
- SimpleModule module = new SimpleModule("Test Anonymization Serializer",
|
|
|
- new Version(0, 0, 0, "TEST"));
|
|
|
- // add various serializers to the module
|
|
|
- module.addSerializer(DataType.class, new DefaultRumenSerializer());
|
|
|
- module.addSerializer(AnonymizableDataType.class,
|
|
|
- new DefaultAnonymizingRumenSerializer(new StatePool(),
|
|
|
- conf));
|
|
|
-
|
|
|
- //TODO Support deprecated and un-supported keys
|
|
|
- testSerializer(jp, "{mapreduce.job.user.name:user0,"
|
|
|
- + "mapred.map.child.java.opts:-Xmx2G -Xms500m}", module);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link ClassName}, serialization and anonymization.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testClassNameSerialization() throws IOException {
|
|
|
- JsonSerializer<?> defaultSerializer = new DefaultRumenSerializer();
|
|
|
-
|
|
|
- JsonSerializer<?> anonymizingSerializer =
|
|
|
- new DefaultAnonymizingRumenSerializer(new StatePool(),
|
|
|
- new Configuration());
|
|
|
-
|
|
|
- // test classname
|
|
|
- ClassName cName = new ClassName(TestRumenAnonymization.class.getName());
|
|
|
- assertEquals("Classname error!", TestRumenAnonymization.class.getName(),
|
|
|
- cName.getValue());
|
|
|
-
|
|
|
- // test classname serialization
|
|
|
- // test with no anonymization
|
|
|
- // test class1
|
|
|
- testSerializer(new ClassName("org.apache.hadoop.Test"),
|
|
|
- "org.apache.hadoop.Test", defaultSerializer);
|
|
|
- // test class2
|
|
|
- testSerializer(new ClassName("org.apache.hadoop.Test2"),
|
|
|
- "org.apache.hadoop.Test2", defaultSerializer);
|
|
|
-
|
|
|
- // test class-name serialization
|
|
|
- // test with anonymization
|
|
|
- // test class1
|
|
|
- testSerializer(new ClassName("org.apache.hadoop.Test1"),
|
|
|
- "class0", anonymizingSerializer);
|
|
|
- // test class2
|
|
|
- testSerializer(new ClassName("org.apache.hadoop.Test2"),
|
|
|
- "class1", anonymizingSerializer);
|
|
|
-
|
|
|
- // test classnames with preserves
|
|
|
- Configuration conf = new Configuration();
|
|
|
- conf.set(ClassName.CLASSNAME_PRESERVE_CONFIG, "org.apache.hadoop.");
|
|
|
- anonymizingSerializer =
|
|
|
- new DefaultAnonymizingRumenSerializer(new StatePool(), conf);
|
|
|
- // test word with prefix
|
|
|
- testSerializer(new ClassName("org.apache.hadoop.Test3"),
|
|
|
- "org.apache.hadoop.Test3", anonymizingSerializer);
|
|
|
- // test word without prefix
|
|
|
- testSerializer(new ClassName("org.apache.hadoop2.Test4"),
|
|
|
- "class0", anonymizingSerializer);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link FileName}.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testFileName() throws IOException {
|
|
|
- // test file on hdfs
|
|
|
- FileName hFile = new FileName("hdfs://testnn:123/user/test.json");
|
|
|
- assertEquals("Filename error!", "hdfs://testnn:123/user/test.json",
|
|
|
- hFile.getValue());
|
|
|
- // test file on local-fs
|
|
|
- hFile = new FileName("file:///user/test.json");
|
|
|
- assertEquals("Filename error!", "file:///user/test.json",
|
|
|
- hFile.getValue());
|
|
|
- // test dir on hdfs
|
|
|
- hFile = new FileName("hdfs://testnn:123/user/");
|
|
|
- assertEquals("Filename error!", "hdfs://testnn:123/user/",
|
|
|
- hFile.getValue());
|
|
|
- // test dir on local-fs
|
|
|
- hFile = new FileName("file:///user/");
|
|
|
- assertEquals("Filename error!", "file:///user/", hFile.getValue());
|
|
|
- // test absolute file
|
|
|
- hFile = new FileName("/user/test/test.json");
|
|
|
- assertEquals("Filename error!", "/user/test/test.json", hFile.getValue());
|
|
|
- // test absolute directory
|
|
|
- hFile = new FileName("/user/test/");
|
|
|
- assertEquals("Filename error!", "/user/test/", hFile.getValue());
|
|
|
- // test relative file
|
|
|
- hFile = new FileName("user/test/test2.json");
|
|
|
- assertEquals("Filename error!", "user/test/test2.json", hFile.getValue());
|
|
|
- // test relative directory
|
|
|
- hFile = new FileName("user/test/");
|
|
|
- assertEquals("Filename error!", "user/test/", hFile.getValue());
|
|
|
- // test absolute file
|
|
|
- hFile = new FileName("user");
|
|
|
- assertEquals("Filename error!", "user", hFile.getValue());
|
|
|
- // test absolute directory
|
|
|
- hFile = new FileName("user/");
|
|
|
- assertEquals("Filename error!", "user/", hFile.getValue());
|
|
|
- hFile = new FileName("./tmp");
|
|
|
- assertEquals("Filename error!","./tmp", hFile.getValue());
|
|
|
- hFile = new FileName("./tmp/");
|
|
|
- assertEquals("Filename error!","./tmp/", hFile.getValue());
|
|
|
- hFile = new FileName("../tmp");
|
|
|
- assertEquals("Filename error!","../tmp", hFile.getValue());
|
|
|
- hFile = new FileName("../tmp/");
|
|
|
- assertEquals("Filename error!","../tmp/", hFile.getValue());
|
|
|
-
|
|
|
- // test comma separated filenames
|
|
|
- // test hdfs filenames, absolute and local-fs filenames
|
|
|
- hFile = new FileName("hdfs://testnn:123/user/test1,"
|
|
|
- + "file:///user/test2,/user/test3");
|
|
|
- assertEquals("Filename error!",
|
|
|
- "hdfs://testnn:123/user/test1,file:///user/test2,/user/test3",
|
|
|
- hFile.getValue());
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link FileName} serialization.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testFileNameSerialization() throws IOException {
|
|
|
- JsonSerializer<?> defaultSerializer = new DefaultRumenSerializer();
|
|
|
-
|
|
|
- // test filename serialization
|
|
|
- // test with no anonymization
|
|
|
- // test a file on hdfs
|
|
|
- testSerializer(new FileName("hdfs://mynn:123/home/user/test.json"),
|
|
|
- "hdfs://mynn:123/home/user/test.json", defaultSerializer);
|
|
|
- // test a file on local-fs
|
|
|
- testSerializer(new FileName("file:///home/user/test.json"),
|
|
|
- "file:///home/user/test.json", defaultSerializer);
|
|
|
- // test directory on hdfs
|
|
|
- testSerializer(new FileName("hdfs://mynn:123/home/user/"),
|
|
|
- "hdfs://mynn:123/home/user/", defaultSerializer);
|
|
|
- // test directory on local fs
|
|
|
- testSerializer(new FileName("file:///home/user/"),
|
|
|
- "file:///home/user/", defaultSerializer);
|
|
|
- // test absolute file
|
|
|
- testSerializer(new FileName("/home/user/test.json"),
|
|
|
- "/home/user/test.json", defaultSerializer);
|
|
|
- // test relative file
|
|
|
- testSerializer(new FileName("home/user/test.json"),
|
|
|
- "home/user/test.json", defaultSerializer);
|
|
|
- // test absolute folder
|
|
|
- testSerializer(new FileName("/home/user/"), "/home/user/",
|
|
|
- defaultSerializer);
|
|
|
- // test relative folder
|
|
|
- testSerializer(new FileName("home/user/"), "home/user/",
|
|
|
- defaultSerializer);
|
|
|
- // relative file
|
|
|
- testSerializer(new FileName("home"), "home", defaultSerializer);
|
|
|
- // relative folder
|
|
|
- testSerializer(new FileName("home/"), "home/", defaultSerializer);
|
|
|
- // absolute file
|
|
|
- testSerializer(new FileName("/home"), "/home", defaultSerializer);
|
|
|
- // absolute folder
|
|
|
- testSerializer(new FileName("/home/"), "/home/", defaultSerializer);
|
|
|
- // relative folder
|
|
|
- testSerializer(new FileName("./tmp"), "./tmp", defaultSerializer);
|
|
|
- testSerializer(new FileName("./tmp/"), "./tmp/", defaultSerializer);
|
|
|
- testSerializer(new FileName("../tmp"), "../tmp", defaultSerializer);
|
|
|
-
|
|
|
- // test comma separated filenames
|
|
|
- // test hdfs filenames, absolute and local-fs filenames
|
|
|
- FileName fileName =
|
|
|
- new FileName("hdfs://testnn:123/user/test1,file:///user/test2,"
|
|
|
- + "/user/test3");
|
|
|
- testSerializer(fileName,
|
|
|
- "hdfs://testnn:123/user/test1,file:///user/test2,/user/test3",
|
|
|
- defaultSerializer);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link FileName} anonymization.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testFileNameAnonymization() throws IOException {
|
|
|
- JsonSerializer<?> anonymizingSerializer =
|
|
|
- new DefaultAnonymizingRumenSerializer(new StatePool(),
|
|
|
- new Configuration());
|
|
|
-
|
|
|
- // test filename serialization
|
|
|
- // test with no anonymization
|
|
|
- // test hdfs file
|
|
|
- testSerializer(new FileName("hdfs://mynn:123/home/user/bob/test.json"),
|
|
|
- "hdfs://host0/home/user/dir0/test.json", anonymizingSerializer);
|
|
|
- // test local-fs file
|
|
|
- testSerializer(new FileName("file:///home/user/alice/test.jar"),
|
|
|
- "file:///home/user/dir1/test.jar", anonymizingSerializer);
|
|
|
- // test hdfs dir
|
|
|
- testSerializer(new FileName("hdfs://mynn:123/home/user/"),
|
|
|
- "hdfs://host0/home/user/", anonymizingSerializer);
|
|
|
- // test local-fs dir
|
|
|
- testSerializer(new FileName("file:///home/user/secret/more-secret/"),
|
|
|
- "file:///home/user/dir2/dir3/", anonymizingSerializer);
|
|
|
- // test absolute filenames
|
|
|
- testSerializer(new FileName("/home/user/top-secret.txt"),
|
|
|
- "/home/user/file0.txt", anonymizingSerializer);
|
|
|
- // test relative filenames
|
|
|
- testSerializer(new FileName("home/user/top-top-secret.zip"),
|
|
|
- "home/user/file1.zip", anonymizingSerializer);
|
|
|
- // test absolute dirnames
|
|
|
- testSerializer(new FileName("/home/user/project1/"),
|
|
|
- "/home/user/dir4/", anonymizingSerializer);
|
|
|
- // test relative filenames
|
|
|
- testSerializer(new FileName("home/user/project1"),
|
|
|
- "home/user/file2", anonymizingSerializer);
|
|
|
- // test absolute dirnames (re-use)
|
|
|
- testSerializer(new FileName("more-secret/"),
|
|
|
- "dir3/", anonymizingSerializer);
|
|
|
- // test relative filenames (re-use)
|
|
|
- testSerializer(new FileName("secret/project1"),
|
|
|
- "dir2/file2", anonymizingSerializer);
|
|
|
- // test absolute filenames (re-use)
|
|
|
- testSerializer(new FileName("/top-secret.txt"),
|
|
|
- "/file0.txt", anonymizingSerializer);
|
|
|
- // test relative filenames (re-use)
|
|
|
- testSerializer(new FileName("top-top-secret.tar"),
|
|
|
- "file1.tar", anonymizingSerializer);
|
|
|
- // test absolute dirname
|
|
|
- testSerializer(new FileName("sensitive-projectname/"),
|
|
|
- "dir5/", anonymizingSerializer);
|
|
|
- // test relative filenames
|
|
|
- testSerializer(new FileName("/real-sensitive-projectname/"),
|
|
|
- "/dir6/", anonymizingSerializer);
|
|
|
- // test absolute filenames
|
|
|
- testSerializer(new FileName("/usernames.xml"),
|
|
|
- "/file3.xml", anonymizingSerializer);
|
|
|
- // test relative filenames
|
|
|
- testSerializer(new FileName("passwords.zip"),
|
|
|
- "file4.zip", anonymizingSerializer);
|
|
|
- // test relative filenames
|
|
|
- testSerializer(new FileName("./tmp"),
|
|
|
- "./tmp", anonymizingSerializer);
|
|
|
- testSerializer(new FileName("./tmp/"),
|
|
|
- "./tmp/", anonymizingSerializer);
|
|
|
- testSerializer(new FileName("../tmp"),
|
|
|
- "../tmp", anonymizingSerializer);
|
|
|
- testSerializer(new FileName("../tmp/"),
|
|
|
- "../tmp/", anonymizingSerializer);
|
|
|
-
|
|
|
- // test comma separated filenames
|
|
|
- // test hdfs filenames, absolute and local-fs filenames
|
|
|
- FileName fileName =
|
|
|
- new FileName("hdfs://mynn:123/home/user/bob/test.json,"
|
|
|
- + "file:///home/user/bob/test.json,/user/alice/test.json");
|
|
|
- testSerializer(fileName,
|
|
|
- "hdfs://host0/home/user/dir0/test.json,file:///home/user/dir0/test.json"
|
|
|
- + ",/user/dir1/test.json",
|
|
|
- anonymizingSerializer);
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link DefaultDataType} serialization.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testDefaultDataTypeSerialization() throws IOException {
|
|
|
- JsonSerializer<?> defaultSerializer = new DefaultRumenSerializer();
|
|
|
-
|
|
|
- // test default data-type
|
|
|
- DefaultDataType dt = new DefaultDataType("test");
|
|
|
- assertEquals("DefaultDataType error!", "test", dt.getValue());
|
|
|
-
|
|
|
- // test default data-type
|
|
|
- // test with no anonymization
|
|
|
- // test data
|
|
|
- testSerializer(new DefaultDataType("test"), "test", defaultSerializer);
|
|
|
- }
|
|
|
-
|
|
|
- // A faked OutputStream which stores the stream content into a StringBuffer.
|
|
|
- private static class MyOutputStream extends OutputStream {
|
|
|
- private StringBuffer data = new StringBuffer();
|
|
|
-
|
|
|
- @Override
|
|
|
- public void write(int b) throws IOException {
|
|
|
- data.append((char)b);
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- public void write(byte[] b) throws IOException {
|
|
|
- data.append(b);
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- public String toString() {
|
|
|
- // remove all the '"' for ease of testing
|
|
|
- return data.toString().trim().replaceAll("\"", "");
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // tests the object serializing using the class of the specified object
|
|
|
- @SuppressWarnings("unchecked")
|
|
|
- private static void testSerializer(Object toBeSerialized, String expData,
|
|
|
- JsonSerializer serializer)
|
|
|
- throws IOException {
|
|
|
- // define a module
|
|
|
- SimpleModule module = new SimpleModule("Test Anonymization Serializer",
|
|
|
- new Version(0, 0, 0, "TEST"));
|
|
|
- // add various serializers to the module
|
|
|
- module.addSerializer(toBeSerialized.getClass(), serializer);
|
|
|
- testSerializer(toBeSerialized, expData, module);
|
|
|
- }
|
|
|
-
|
|
|
- // tests the object serializing using the specified class
|
|
|
- private static void testSerializer(Object toBeSerialized, String expData,
|
|
|
- SimpleModule module)
|
|
|
- throws IOException {
|
|
|
- // define a custom generator
|
|
|
- ObjectMapper outMapper = new ObjectMapper();
|
|
|
-
|
|
|
- // register the module
|
|
|
- outMapper.registerModule(module);
|
|
|
-
|
|
|
- // get the json factory
|
|
|
- JsonFactory outFactory = outMapper.getJsonFactory();
|
|
|
- // define a fake output stream which will cache the data
|
|
|
- MyOutputStream output = new MyOutputStream();
|
|
|
- // define the json output generator
|
|
|
- JsonGenerator outGen =
|
|
|
- outFactory.createJsonGenerator(output, JsonEncoding.UTF8);
|
|
|
-
|
|
|
- // serialize the object
|
|
|
- outGen.writeObject(toBeSerialized);
|
|
|
- //serializer.serialize(toBeSerialized, outGen, null);
|
|
|
-
|
|
|
- // close the json generator so that it flushes out the data to the output
|
|
|
- // stream
|
|
|
- outGen.close();
|
|
|
-
|
|
|
- assertEquals("Serialization failed!", expData, output.toString());
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link DefaultRumenSerializer}.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testDefaultDataSerializers() throws Exception {
|
|
|
- JsonSerializer<?> defaultSer = new DefaultRumenSerializer();
|
|
|
- // test default data-type
|
|
|
- // test with no anonymization
|
|
|
- // test data
|
|
|
- testSerializer(new DefaultDataType("test"), "test", defaultSer);
|
|
|
- }
|
|
|
-
|
|
|
- @Test
|
|
|
- public void testBlockingDataSerializers() throws Exception {
|
|
|
- JsonSerializer<?> blockingSerializer = new BlockingSerializer();
|
|
|
-
|
|
|
- // test string serializer
|
|
|
- testSerializer("username:password", "null", blockingSerializer);
|
|
|
- }
|
|
|
-
|
|
|
- @Test
|
|
|
- public void testObjectStringDataSerializers() throws Exception {
|
|
|
- JsonSerializer<?> objectStringSerializer = new ObjectStringSerializer<ID>();
|
|
|
- // test job/task/attempt id serializer
|
|
|
- // test job-id
|
|
|
- JobID jid = JobID.forName("job_1_1");
|
|
|
- testSerializer(jid, jid.toString(), objectStringSerializer);
|
|
|
- // test task-id
|
|
|
- TaskID tid = new TaskID(jid, TaskType.MAP, 1);
|
|
|
- testSerializer(tid, tid.toString(), objectStringSerializer);
|
|
|
- // test attempt-id
|
|
|
- TaskAttemptID aid = new TaskAttemptID(tid, 0);
|
|
|
- testSerializer(aid, aid.toString(), objectStringSerializer);
|
|
|
- }
|
|
|
-
|
|
|
- // test anonymizer
|
|
|
- @Test
|
|
|
- public void testRumenAnonymization() throws Exception {
|
|
|
- Configuration conf = new Configuration();
|
|
|
-
|
|
|
- // Run a MR job
|
|
|
- // create a MR cluster
|
|
|
- conf.setInt(TTConfig.TT_MAP_SLOTS, 1);
|
|
|
- conf.setInt(TTConfig.TT_REDUCE_SLOTS, 1);
|
|
|
-
|
|
|
- MiniDFSCluster dfsCluster = null;
|
|
|
- MiniMRCluster mrCluster = null;
|
|
|
-
|
|
|
- // local filesystem for running TraceBuilder
|
|
|
- FileSystem lfs = FileSystem.getLocal(conf);
|
|
|
- Path rootTempDir =
|
|
|
- new Path(System.getProperty("test.build.data", "/tmp"));
|
|
|
-
|
|
|
- Path tempDir = new Path(rootTempDir, "testRumenAnonymization");
|
|
|
- tempDir = lfs.makeQualified(tempDir);
|
|
|
- lfs.delete(tempDir, true);
|
|
|
-
|
|
|
- try {
|
|
|
- dfsCluster = new MiniDFSCluster(conf, 1, true, null);
|
|
|
- String[] racks = new String[] {"/rack123.myorg.com",
|
|
|
- "/rack456.myorg.com"};
|
|
|
- String[] hosts = new String[] {"host1230.myorg.com",
|
|
|
- "host4560.myorg.com"};
|
|
|
- mrCluster =
|
|
|
- new MiniMRCluster(2, dfsCluster.getFileSystem().getUri().toString(),
|
|
|
- 1, racks, hosts, new JobConf(conf));
|
|
|
-
|
|
|
- // run a job
|
|
|
- Path inDir = new Path("secret-input");
|
|
|
- Path outDir = new Path("secret-output");
|
|
|
-
|
|
|
- JobConf jConf = mrCluster.createJobConf();
|
|
|
- // add some usr sensitive data in the job conf
|
|
|
- jConf.set("user-secret-code", "abracadabra");
|
|
|
-
|
|
|
- jConf.setJobName("top-secret");
|
|
|
- // construct a job with 1 map and 1 reduce task.
|
|
|
- Job job = MapReduceTestUtil.createJob(jConf, inDir, outDir, 2, 2);
|
|
|
- // wait for the job to complete
|
|
|
- job.waitForCompletion(false);
|
|
|
-
|
|
|
- assertTrue("Job failed", job.isSuccessful());
|
|
|
-
|
|
|
- JobID id = job.getJobID();
|
|
|
- Cluster cluster = new Cluster(jConf);
|
|
|
- String user = cluster.getAllJobStatuses()[0].getUsername();
|
|
|
-
|
|
|
- // get the jobhistory filepath
|
|
|
- Path jhPath =
|
|
|
- new Path(mrCluster.getJobTrackerRunner().getJobTracker()
|
|
|
- .getJobHistoryDir());
|
|
|
- Path inputLogPath = JobHistory.getJobHistoryFile(jhPath, id, user);
|
|
|
- Path inputConfPath = JobHistory.getConfFile(jhPath, id);
|
|
|
- // wait for 10 secs for the jobhistory file to move into the done folder
|
|
|
- FileSystem fs = inputLogPath.getFileSystem(jConf);
|
|
|
- for (int i = 0; i < 100; ++i) {
|
|
|
- if (fs.exists(inputLogPath)) {
|
|
|
- break;
|
|
|
- }
|
|
|
- TimeUnit.MILLISECONDS.wait(100);
|
|
|
- }
|
|
|
-
|
|
|
- assertTrue("Missing job history file", fs.exists(inputLogPath));
|
|
|
-
|
|
|
- // run trace builder on the job history logs
|
|
|
- Path goldTraceFilename = new Path(tempDir, "trace.json");
|
|
|
- Path goldTopologyFilename = new Path(tempDir, "topology.json");
|
|
|
-
|
|
|
- // build the trace-builder command line args
|
|
|
- String[] args = new String[] {goldTraceFilename.toString(),
|
|
|
- goldTopologyFilename.toString(),
|
|
|
- inputLogPath.toString(),
|
|
|
- inputConfPath.toString()};
|
|
|
- Tool analyzer = new TraceBuilder();
|
|
|
- int result = ToolRunner.run(analyzer, args);
|
|
|
- assertEquals("Non-zero exit", 0, result);
|
|
|
-
|
|
|
- // anonymize the job trace
|
|
|
- Path anonymizedTraceFilename = new Path(tempDir, "trace-anonymized.json");
|
|
|
- Path anonymizedClusterTopologyFilename =
|
|
|
- new Path(tempDir, "topology-anonymized.json");
|
|
|
- args = new String[] {"-trace", goldTraceFilename.toString(),
|
|
|
- anonymizedTraceFilename.toString(),
|
|
|
- "-topology", goldTopologyFilename.toString(),
|
|
|
- anonymizedClusterTopologyFilename.toString()};
|
|
|
- Tool anonymizer = new Anonymizer();
|
|
|
- result = ToolRunner.run(anonymizer, args);
|
|
|
- assertEquals("Non-zero exit", 0, result);
|
|
|
-
|
|
|
- JobTraceReader reader = new JobTraceReader(anonymizedTraceFilename, conf);
|
|
|
- LoggedJob anonymizedJob = reader.getNext();
|
|
|
- reader.close(); // close the reader as we need only 1 job
|
|
|
- // test
|
|
|
- // user-name
|
|
|
- String currentUser = UserGroupInformation.getCurrentUser().getUserName();
|
|
|
- assertFalse("Username not anonymized!",
|
|
|
- currentUser.equals(anonymizedJob.getUser().getValue()));
|
|
|
- // jobid
|
|
|
- assertEquals("JobID mismatch!",
|
|
|
- id.toString(), anonymizedJob.getJobID().toString());
|
|
|
- // queue-name
|
|
|
- assertFalse("Queuename mismatch!",
|
|
|
- "default".equals(anonymizedJob.getQueue().getValue()));
|
|
|
- // job-name
|
|
|
- assertFalse("Jobname mismatch!",
|
|
|
- "top-secret".equals(anonymizedJob.getJobName().getValue()));
|
|
|
-
|
|
|
- // job properties
|
|
|
- for (Map.Entry<Object, Object> entry :
|
|
|
- anonymizedJob.getJobProperties().getValue().entrySet()) {
|
|
|
- assertFalse("User sensitive configuration key not anonymized",
|
|
|
- entry.getKey().toString().equals("user-secret-code"));
|
|
|
- assertFalse("User sensitive data not anonymized",
|
|
|
- entry.getValue().toString().contains(currentUser));
|
|
|
- assertFalse("User sensitive data not anonymized",
|
|
|
- entry.getValue().toString().contains("secret"));
|
|
|
- }
|
|
|
-
|
|
|
- // test map tasks
|
|
|
- testTasks(anonymizedJob.getMapTasks(), id, TaskType.MAP);
|
|
|
-
|
|
|
- // test reduce tasks
|
|
|
- testTasks(anonymizedJob.getReduceTasks(), id, TaskType.REDUCE);
|
|
|
-
|
|
|
- // test other tasks
|
|
|
- testTasks(anonymizedJob.getOtherTasks(), id, null);
|
|
|
-
|
|
|
- // test the anonymized cluster topology file
|
|
|
- ClusterTopologyReader cReader =
|
|
|
- new ClusterTopologyReader(anonymizedClusterTopologyFilename, conf);
|
|
|
- LoggedNetworkTopology loggedNetworkTopology = cReader.get();
|
|
|
- // test the cluster topology
|
|
|
- testClusterTopology(loggedNetworkTopology, 0, "myorg");
|
|
|
- } finally {
|
|
|
- // shutdown and cleanup
|
|
|
- if (mrCluster != null) {
|
|
|
- mrCluster.shutdown();
|
|
|
- }
|
|
|
-
|
|
|
- if (dfsCluster != null) {
|
|
|
- dfsCluster.formatDataNodeDirs();
|
|
|
- dfsCluster.shutdown();
|
|
|
- }
|
|
|
- lfs.delete(tempDir, true);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // test task level details lije
|
|
|
- // - taskid
|
|
|
- // - locality info
|
|
|
- // - attempt details
|
|
|
- // - attempt execution hostname
|
|
|
- private static void testTasks(List<LoggedTask> tasks, JobID id,
|
|
|
- TaskType type) {
|
|
|
- int index = 0;
|
|
|
- for (LoggedTask task : tasks) {
|
|
|
- // generate the expected task id for this task
|
|
|
- if (type != null) {
|
|
|
- TaskID tid = new TaskID(id, type, index++);
|
|
|
- assertEquals("TaskID mismatch!",
|
|
|
- tid.toString(), task.getTaskID().toString());
|
|
|
- }
|
|
|
-
|
|
|
- // check locality information
|
|
|
- if (task.getPreferredLocations() != null) {
|
|
|
- for (LoggedLocation loc : task.getPreferredLocations()) {
|
|
|
- for (NodeName name : loc.getLayers()) {
|
|
|
- assertFalse("Hostname mismatch!",
|
|
|
- name.getValue().contains("myorg"));
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // check execution host
|
|
|
- for (LoggedTaskAttempt attempt : task.getAttempts()) {
|
|
|
- // generate the expected task id for this task
|
|
|
- TaskAttemptID aid = new TaskAttemptID(task.getTaskID(), 0);
|
|
|
- assertEquals("TaskAttemptID mismatch!",
|
|
|
- aid.toString(), attempt.getAttemptID().toString());
|
|
|
-
|
|
|
- assertNotNull("Hostname null!", attempt.getHostName());
|
|
|
- assertFalse("Hostname mismatch!",
|
|
|
- attempt.getHostName().getValue().contains("myorg"));
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // tests the logged network topology
|
|
|
- private static void testClusterTopology(LoggedNetworkTopology topology,
|
|
|
- int level, String bannedString) {
|
|
|
- assertFalse("Cluster topology test failed!",
|
|
|
- topology.getName().getValue().contains(bannedString));
|
|
|
- if (level == 0) {
|
|
|
- assertEquals("Level-1 data mismatch!",
|
|
|
- "<root>", topology.getName().getValue());
|
|
|
- } else if (level == 1) {
|
|
|
- assertTrue("Level-2 data mismatch!",
|
|
|
- topology.getName().getValue().contains("rack"));
|
|
|
- assertFalse("Level-2 data mismatch!",
|
|
|
- topology.getName().getValue().contains("host"));
|
|
|
- } else {
|
|
|
- assertTrue("Level-2 data mismatch!",
|
|
|
- topology.getName().getValue().contains("host"));
|
|
|
- assertFalse("Level-2 data mismatch!",
|
|
|
- topology.getName().getValue().contains("rack"));
|
|
|
- }
|
|
|
-
|
|
|
- // if the current node is a rack, then test the nodes under it
|
|
|
- if (topology.getChildren() != null) {
|
|
|
- for (LoggedNetworkTopology child : topology.getChildren()) {
|
|
|
- testClusterTopology(child, level + 1, bannedString);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- @Test
|
|
|
- public void testCLI() throws Exception {
|
|
|
- Configuration conf = new Configuration();
|
|
|
- FileSystem lfs = FileSystem.getLocal(conf);
|
|
|
- Path rootTempDir =
|
|
|
- new Path(System.getProperty("test.build.data", "/tmp"));
|
|
|
-
|
|
|
- Path tempDir = new Path(rootTempDir, "testCLI");
|
|
|
- tempDir = lfs.makeQualified(tempDir);
|
|
|
- lfs.delete(tempDir, true);
|
|
|
-
|
|
|
- // test no args
|
|
|
- String[] args = new String[] {};
|
|
|
- testAnonymizerCLI(args, -1);
|
|
|
-
|
|
|
- // test with wrong args
|
|
|
- args = new String[] {"-test"};
|
|
|
- testAnonymizerCLI(args, -1);
|
|
|
-
|
|
|
- args = new String[] {"-trace"};
|
|
|
- testAnonymizerCLI(args, -1);
|
|
|
-
|
|
|
- args = new String[] {"-topology"};
|
|
|
- testAnonymizerCLI(args, -1);
|
|
|
-
|
|
|
- args = new String[] {"-trace -topology"};
|
|
|
- testAnonymizerCLI(args, -1);
|
|
|
-
|
|
|
- Path testTraceInputFilename = new Path(tempDir, "trace-in.json");
|
|
|
- args = new String[] {"-trace", testTraceInputFilename.toString()};
|
|
|
- testAnonymizerCLI(args, -1);
|
|
|
-
|
|
|
- Path testTraceOutputFilename = new Path(tempDir, "trace-out.json");
|
|
|
- args = new String[] {"-trace", testTraceInputFilename.toString(),
|
|
|
- testTraceOutputFilename.toString()};
|
|
|
- testAnonymizerCLI(args, -1);
|
|
|
-
|
|
|
- OutputStream out = lfs.create(testTraceInputFilename);
|
|
|
- out.write("{\n}".getBytes());
|
|
|
- out.close();
|
|
|
- args = new String[] {"-trace", testTraceInputFilename.toString(),
|
|
|
- testTraceOutputFilename.toString()};
|
|
|
- testAnonymizerCLI(args, 0);
|
|
|
-
|
|
|
- Path testToplogyInputFilename = new Path(tempDir, "topology-in.json");
|
|
|
- args = new String[] {"-topology", testToplogyInputFilename.toString()};
|
|
|
- testAnonymizerCLI(args, -1);
|
|
|
-
|
|
|
- Path testTopologyOutputFilename = new Path(tempDir, "topology-out.json");
|
|
|
- args = new String[] {"-topology", testToplogyInputFilename.toString(),
|
|
|
- testTopologyOutputFilename.toString()};
|
|
|
- testAnonymizerCLI(args, -1);
|
|
|
-
|
|
|
- out = lfs.create(testToplogyInputFilename);
|
|
|
- out.write("{\n}".getBytes());
|
|
|
- out.close();
|
|
|
- args = new String[] {"-topology", testToplogyInputFilename.toString(),
|
|
|
- testTopologyOutputFilename.toString()};
|
|
|
- testAnonymizerCLI(args, 0);
|
|
|
-
|
|
|
- args = new String[] {"-trace", testTraceInputFilename.toString(),
|
|
|
- "-topology", testToplogyInputFilename.toString()};
|
|
|
- testAnonymizerCLI(args, -1);
|
|
|
-
|
|
|
- args = new String[] {"-trace", testTraceInputFilename.toString(),
|
|
|
- testTraceOutputFilename.toString(),
|
|
|
- "-topology", testToplogyInputFilename.toString(),
|
|
|
- testTopologyOutputFilename.toString()};
|
|
|
- testAnonymizerCLI(args, 0);
|
|
|
- }
|
|
|
-
|
|
|
- // tests the Anonymizer CLI via the Tools interface
|
|
|
- private static void testAnonymizerCLI(String[] args, int eExitCode)
|
|
|
- throws Exception {
|
|
|
- Anonymizer anonymizer = new Anonymizer();
|
|
|
-
|
|
|
- int exitCode = ToolRunner.run(anonymizer, args);
|
|
|
- assertEquals("Exit code mismatch", eExitCode, exitCode);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link StatePool}'s reload and persistence feature.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testStatePool() throws Exception {
|
|
|
- Configuration conf = new Configuration();
|
|
|
- FileSystem lfs = FileSystem.getLocal(conf);
|
|
|
- Path rootTempDir =
|
|
|
- new Path(System.getProperty("test.build.data", "/tmp"));
|
|
|
-
|
|
|
- Path tempDir = new Path(rootTempDir, "testStatePool");
|
|
|
- tempDir = lfs.makeQualified(tempDir);
|
|
|
- lfs.delete(tempDir, true);
|
|
|
-
|
|
|
- // set the state dir
|
|
|
- conf.set(StatePool.DIR_CONFIG, tempDir.toString());
|
|
|
-
|
|
|
- StatePool pool = new StatePool();
|
|
|
- // test reload, persist and dir config
|
|
|
- // test with no reload and persist
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- // test with reload and/or persist enabled with no dir
|
|
|
- assertNull("Default state pool error",
|
|
|
- pool.getState(MyState.class));
|
|
|
-
|
|
|
- // try persisting
|
|
|
- pool.persist();
|
|
|
- assertFalse("State pool persisted when disabled", lfs.exists(tempDir));
|
|
|
-
|
|
|
- // test wrongly configured state-pool
|
|
|
- conf.setBoolean(StatePool.RELOAD_CONFIG, true);
|
|
|
- conf.unset(StatePool.DIR_CONFIG);
|
|
|
- pool = new StatePool();
|
|
|
- boolean success = true;
|
|
|
- try {
|
|
|
- pool.initialize(conf);
|
|
|
- } catch (Exception e) {
|
|
|
- success = false;
|
|
|
- }
|
|
|
- assertFalse("State pool bad configuration succeeded", success);
|
|
|
-
|
|
|
- // test wrongly configured state-pool
|
|
|
- conf.setBoolean(StatePool.RELOAD_CONFIG, false);
|
|
|
- conf.setBoolean(StatePool.PERSIST_CONFIG, true);
|
|
|
- pool = new StatePool();
|
|
|
- success = true;
|
|
|
- try {
|
|
|
- pool.initialize(conf);
|
|
|
- } catch (Exception e) {
|
|
|
- success = false;
|
|
|
- }
|
|
|
- assertFalse("State manager bad configuration succeeded", success);
|
|
|
-
|
|
|
-
|
|
|
- // test persistence
|
|
|
- conf.setBoolean(StatePool.RELOAD_CONFIG, false);
|
|
|
- conf.set(StatePool.DIR_CONFIG, tempDir.toString());
|
|
|
- conf.setBoolean(StatePool.PERSIST_CONFIG, true);
|
|
|
- pool = new StatePool();
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- // add states to the state pool
|
|
|
- MyState myState = new MyState();
|
|
|
- pool.addState(MyState.class, myState);
|
|
|
- myState.setState("test-1");
|
|
|
- // try persisting
|
|
|
- pool.persist();
|
|
|
- assertTrue("State pool persisted when enabled", lfs.exists(tempDir));
|
|
|
- assertEquals("State pool persisted when enabled",
|
|
|
- 1, lfs.listStatus(tempDir).length);
|
|
|
-
|
|
|
- // reload
|
|
|
- conf.setBoolean(StatePool.RELOAD_CONFIG, true);
|
|
|
- conf.set(StatePool.DIR_CONFIG, tempDir.toString());
|
|
|
- conf.setBoolean(StatePool.PERSIST_CONFIG, true);
|
|
|
- pool = new StatePool();
|
|
|
- pool.initialize(conf);
|
|
|
- MyState pState =
|
|
|
- (MyState) pool.getState(MyState.class);
|
|
|
- assertEquals("State pool persistence/reload failed", "test-1",
|
|
|
- pState.getState());
|
|
|
-
|
|
|
- // try persisting with no state change
|
|
|
- pool.persist();
|
|
|
- assertEquals("State pool persisted when disabled",
|
|
|
- 1, lfs.listStatus(tempDir).length);
|
|
|
-
|
|
|
- // modify the state of the pool and check persistence
|
|
|
- pState.setUpdated(true);
|
|
|
- pool.persist();
|
|
|
- assertEquals("State pool persisted when disabled",
|
|
|
- 2, lfs.listStatus(tempDir).length);
|
|
|
-
|
|
|
- // delete the temp directory if everything goes fine
|
|
|
- lfs.delete(tempDir, true);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test state.
|
|
|
- */
|
|
|
- static class MyState implements State {
|
|
|
- private boolean updated = false;
|
|
|
- private String state = "null";
|
|
|
-
|
|
|
- @Override
|
|
|
- @JsonIgnore
|
|
|
- public String getName() {
|
|
|
- return "test";
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- public void setName(String name) {
|
|
|
- // for now, simply assert since this class has a hardcoded name
|
|
|
- if (!getName().equals(name)) {
|
|
|
- throw new RuntimeException("State name mismatch! Expected '"
|
|
|
- + getName() + "' but found '" + name + "'.");
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- public void setState(String state) {
|
|
|
- this.state = state;
|
|
|
- }
|
|
|
-
|
|
|
- public String getState() {
|
|
|
- return state;
|
|
|
- }
|
|
|
-
|
|
|
- void setUpdated(boolean up) {
|
|
|
- this.updated = up;
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- @JsonIgnore
|
|
|
- public boolean isUpdated() {
|
|
|
- return updated;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- @SuppressWarnings("unchecked")
|
|
|
- private static String getValueFromDataType(Object object) {
|
|
|
- DataType<String> dt = (DataType<String>) object;
|
|
|
- return dt.getValue();
|
|
|
- }
|
|
|
-
|
|
|
- @Test
|
|
|
- public void testJobPropertiesParser() {
|
|
|
- // test default parser
|
|
|
- Properties properties = new Properties();
|
|
|
- Configuration conf = new Configuration();
|
|
|
- JobProperties jp = new JobProperties(properties);
|
|
|
- assertEquals("Job Properties (default filter) store error",
|
|
|
- 0, jp.getAnonymizedValue(null, conf).size());
|
|
|
-
|
|
|
- // define key-value pairs for job configuration
|
|
|
- String key1 = "test-key";
|
|
|
- String value1 = "test-value";
|
|
|
- properties.put(key1, value1); // user config
|
|
|
- String key2 = MRJobConfig.USER_NAME;
|
|
|
- String value2 = "bob";
|
|
|
- properties.put(key2, value2); // job config
|
|
|
- String key3 = JobConf.MAPRED_MAP_TASK_JAVA_OPTS;
|
|
|
- String value3 = "-Xmx1G";
|
|
|
- properties.put(key3, value3); // deprecated
|
|
|
- String key4 = MRJobConfig.REDUCE_JAVA_OPTS;
|
|
|
- String value4 = "-Xms100m";
|
|
|
- properties.put(key4, value4);
|
|
|
-
|
|
|
- jp = new JobProperties(properties);
|
|
|
-
|
|
|
- // Configure the default parser
|
|
|
- conf.set(JobProperties.PARSERS_CONFIG_KEY,
|
|
|
- DefaultJobPropertiesParser.class.getName());
|
|
|
- // anonymize
|
|
|
- Properties defaultProp = jp.getAnonymizedValue(null, conf);
|
|
|
- assertEquals("Job Properties (all-pass filter) store error",
|
|
|
- 4, defaultProp.size());
|
|
|
- assertEquals("Job Properties (default filter) key#1 error", value1,
|
|
|
- getValueFromDataType(defaultProp.get(key1)));
|
|
|
- assertEquals("Job Properties (default filter) key#2 error", value2,
|
|
|
- getValueFromDataType(defaultProp.get(key2)));
|
|
|
- assertEquals("Job Properties (default filter) key#3 error", value3,
|
|
|
- getValueFromDataType(defaultProp.get(key3)));
|
|
|
- assertEquals("Job Properties (default filter) key#4 error", value4,
|
|
|
- getValueFromDataType(defaultProp.get(key4)));
|
|
|
-
|
|
|
- // test MR parser
|
|
|
- conf.set(JobProperties.PARSERS_CONFIG_KEY,
|
|
|
- MapReduceJobPropertiesParser.class.getName());
|
|
|
- // anonymize
|
|
|
- Properties filteredProp = jp.getAnonymizedValue(null, conf);
|
|
|
- assertEquals("Job Properties (MR filter) store error",
|
|
|
- 3, filteredProp.size());
|
|
|
- assertNull("Job Properties (MR filter) key#1 error",
|
|
|
- filteredProp.get(key1));
|
|
|
- assertEquals("Job Properties (MR filter) key#2 error", value2,
|
|
|
- getValueFromDataType(filteredProp.get(key2)));
|
|
|
- assertEquals("Job Properties (MR filter) key#3 error", value3,
|
|
|
- getValueFromDataType(filteredProp.get(key3)));
|
|
|
- assertEquals("Job Properties (MR filter) key#4 error", value4,
|
|
|
- getValueFromDataType(filteredProp.get(key4)));
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link WordListAnonymizerUtility}. Test various features like
|
|
|
- * - test known words
|
|
|
- * - test known suffix
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testWordListBasedAnonymizer() {
|
|
|
- String[] knownSuffixes = new String[] {".1", ".2", ".3", ".4"};
|
|
|
-
|
|
|
- // test with valid suffix
|
|
|
- assertTrue("suffix test#0 failed!",
|
|
|
- WordListAnonymizerUtility.hasSuffix("a.1", knownSuffixes));
|
|
|
- String split[] =
|
|
|
- WordListAnonymizerUtility.extractSuffix("a.1", knownSuffixes);
|
|
|
- assertEquals("suffix test#1 failed!", 2, split.length);
|
|
|
- assertEquals("suffix test#2 failed!", "a", split[0]);
|
|
|
- assertEquals("suffix test#3 failed!", ".1", split[1]);
|
|
|
-
|
|
|
- // test with valid suffix
|
|
|
- assertTrue("suffix test#0 failed!",
|
|
|
- WordListAnonymizerUtility.hasSuffix("a.1", knownSuffixes));
|
|
|
- split =
|
|
|
- WordListAnonymizerUtility.extractSuffix("/a/b.2", knownSuffixes);
|
|
|
- assertEquals("suffix test#0 failed!", 2, split.length);
|
|
|
- assertEquals("suffix test#1 failed!", "/a/b", split[0]);
|
|
|
- assertEquals("suffix test#2 failed!", ".2", split[1]);
|
|
|
-
|
|
|
- // test with invalid suffix
|
|
|
- assertFalse("suffix test#0 failed!",
|
|
|
- WordListAnonymizerUtility.hasSuffix("a.b", knownSuffixes));
|
|
|
-
|
|
|
- boolean failed = false;
|
|
|
- try {
|
|
|
- split = WordListAnonymizerUtility.extractSuffix("a.b", knownSuffixes);
|
|
|
- } catch (Exception e) {
|
|
|
- failed = true;
|
|
|
- }
|
|
|
- assertTrue("Exception expected!", failed);
|
|
|
-
|
|
|
- String[] knownWords = new String[] {"a", "b"};
|
|
|
-
|
|
|
- // test with valid data
|
|
|
- assertTrue("data test#0 failed!",
|
|
|
- WordListAnonymizerUtility.isKnownData("a", knownWords));
|
|
|
- // test with valid data
|
|
|
- assertTrue("data test#1 failed!",
|
|
|
- WordListAnonymizerUtility.isKnownData("b", knownWords));
|
|
|
- // test with invalid data
|
|
|
- assertFalse("data test#2 failed!",
|
|
|
- WordListAnonymizerUtility.isKnownData("c", knownWords));
|
|
|
-
|
|
|
- // test with valid known word
|
|
|
- assertTrue("data test#3 failed!",
|
|
|
- WordListAnonymizerUtility.isKnownData("job"));
|
|
|
- // test with invalid known word
|
|
|
- assertFalse("data test#4 failed!",
|
|
|
- WordListAnonymizerUtility.isKnownData("bob"));
|
|
|
-
|
|
|
- // test numeric data
|
|
|
- assertFalse("Numeric test failed!",
|
|
|
- WordListAnonymizerUtility.needsAnonymization("123"));
|
|
|
- // test numeric data (unsupported)
|
|
|
- assertTrue("Numeric test failed!",
|
|
|
- WordListAnonymizerUtility.needsAnonymization("123.456"));
|
|
|
- // test text data
|
|
|
- assertTrue("Text test failed!",
|
|
|
- WordListAnonymizerUtility.needsAnonymization("123abc"));
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link WordList} features like
|
|
|
- * - add words
|
|
|
- * - index
|
|
|
- * - contains
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testWordList() throws Exception {
|
|
|
- // test features with fresh state
|
|
|
- WordList wordList = new WordList();
|
|
|
- assertFalse("Word list state incorrect", wordList.isUpdated());
|
|
|
-
|
|
|
- // add some special word
|
|
|
- String test = "abbracadabra";
|
|
|
- wordList.add(test);
|
|
|
- assertTrue("Word list failed to store", wordList.contains(test));
|
|
|
- assertEquals("Word list index failed", 0, wordList.indexOf(test));
|
|
|
- assertEquals("Word list size failed", 1, wordList.getSize());
|
|
|
- assertTrue("Word list state incorrect", wordList.isUpdated());
|
|
|
-
|
|
|
- // add already added word
|
|
|
- wordList.add(test);
|
|
|
- assertEquals("Word list index failed", 0, wordList.indexOf(test));
|
|
|
- assertEquals("Word list size failed", 1, wordList.getSize());
|
|
|
- assertTrue("Word list state incorrect", wordList.isUpdated());
|
|
|
-
|
|
|
- String test2 = "hakuna-matata";
|
|
|
- wordList.add(test2);
|
|
|
- assertTrue("Word list failed to store", wordList.contains(test2));
|
|
|
- assertEquals("Word list index failed", 1, wordList.indexOf(test2));
|
|
|
- assertEquals("Word list size failed", 2, wordList.getSize());
|
|
|
- assertTrue("Word list state incorrect", wordList.isUpdated());
|
|
|
-
|
|
|
- // test persistence
|
|
|
- Configuration conf = new Configuration();
|
|
|
- FileSystem lfs = FileSystem.getLocal(conf);
|
|
|
- Path rootTempDir =
|
|
|
- new Path(System.getProperty("test.build.data", "/tmp"));
|
|
|
-
|
|
|
- Path tempDir = new Path(rootTempDir, "testWordList");
|
|
|
- tempDir = lfs.makeQualified(tempDir);
|
|
|
- lfs.delete(tempDir, true);
|
|
|
-
|
|
|
- // define a state pool to help persist the wordlist
|
|
|
- StatePool pool = new StatePool();
|
|
|
-
|
|
|
- try {
|
|
|
- // set the persistence directory
|
|
|
- conf.set(StatePool.DIR_CONFIG, tempDir.toString());
|
|
|
- conf.setBoolean(StatePool.PERSIST_CONFIG, true);
|
|
|
-
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- // add the wordlist to the pool
|
|
|
- pool.addState(getClass(), wordList);
|
|
|
-
|
|
|
- pool.persist();
|
|
|
-
|
|
|
- // now clear the pool state
|
|
|
- pool = new StatePool();
|
|
|
-
|
|
|
- // set reload to true
|
|
|
- conf.setBoolean(StatePool.RELOAD_CONFIG, true);
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- State state = pool.getState(getClass());
|
|
|
- assertNotNull("Missing state!", state);
|
|
|
- assertEquals("Incorrect state class!", WordList.class, state.getClass());
|
|
|
- WordList pList = (WordList) state;
|
|
|
-
|
|
|
- // check size
|
|
|
- assertEquals("Word list size on reload failed", 2, pList.getSize());
|
|
|
- assertFalse("Word list state incorrect", pList.isUpdated());
|
|
|
-
|
|
|
- // add already added word
|
|
|
- pList.add(test);
|
|
|
- assertEquals("Word list index on reload failed", 0, pList.indexOf(test));
|
|
|
- assertEquals("Word list size on reload failed", 2, pList.getSize());
|
|
|
- assertFalse("Word list state on reload incorrect", pList.isUpdated());
|
|
|
-
|
|
|
- String test3 = "disco-dancer";
|
|
|
- assertFalse("Word list failed to after reload", pList.contains(test3));
|
|
|
- pList.add(test3);
|
|
|
- assertTrue("Word list failed to store on reload", pList.contains(test3));
|
|
|
- assertEquals("Word list index on reload failed", 2, pList.indexOf(test3));
|
|
|
- assertEquals("Word list size on reload failed", 3, pList.getSize());
|
|
|
- assertTrue("Word list state on reload incorrect", pList.isUpdated());
|
|
|
-
|
|
|
- // test previously added (persisted) word
|
|
|
- assertTrue("Word list failed to store on reload", pList.contains(test2));
|
|
|
- assertEquals("Word list index on reload failed", 1, pList.indexOf(test2));
|
|
|
- } finally {
|
|
|
- lfs.delete(tempDir, true);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link FileName#FileNameState} persistence with directories only.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testFileNameStateWithDir() throws Exception {
|
|
|
- // test persistence
|
|
|
- Configuration conf = new Configuration();
|
|
|
- FileSystem lfs = FileSystem.getLocal(conf);
|
|
|
- Path rootTempDir =
|
|
|
- new Path(System.getProperty("test.build.data", "/tmp"));
|
|
|
-
|
|
|
- Path tempDir = new Path(rootTempDir, "testFileNameStateWithDir");
|
|
|
- tempDir = lfs.makeQualified(tempDir);
|
|
|
- lfs.delete(tempDir, true);
|
|
|
-
|
|
|
- // set the persistence directory
|
|
|
- conf.set(StatePool.DIR_CONFIG, tempDir.toString());
|
|
|
- conf.setBoolean(StatePool.PERSIST_CONFIG, true);
|
|
|
-
|
|
|
- // define a state pool to help persist the dirs
|
|
|
- StatePool pool = new StatePool();
|
|
|
-
|
|
|
- FileNameState fState = new FileNameState();
|
|
|
-
|
|
|
- // define the directory names
|
|
|
- String test1 = "test";
|
|
|
- String test2 = "home";
|
|
|
-
|
|
|
- // test dir only
|
|
|
- WordList dirState = new WordList("dir");
|
|
|
- dirState.add(test1);
|
|
|
- dirState.add(test2);
|
|
|
-
|
|
|
- // set the directory state
|
|
|
- fState.setDirectoryState(dirState);
|
|
|
-
|
|
|
- try {
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- // add the wordlist to the pool
|
|
|
- pool.addState(getClass(), fState);
|
|
|
-
|
|
|
- // persist the state
|
|
|
- pool.persist();
|
|
|
-
|
|
|
- // now clear the pool state
|
|
|
- pool = new StatePool();
|
|
|
-
|
|
|
- // set reload to true
|
|
|
- conf.setBoolean(StatePool.RELOAD_CONFIG, true);
|
|
|
-
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- State state = pool.getState(getClass());
|
|
|
- assertNotNull("Missing state!", state);
|
|
|
- assertEquals("Incorrect state class!",
|
|
|
- FileNameState.class, state.getClass());
|
|
|
- FileNameState newFState = (FileNameState) state;
|
|
|
-
|
|
|
- // check the state contents
|
|
|
- WordList newStateWordList = newFState.getDirectoryState();
|
|
|
- assertTrue("File state failed to store on reload",
|
|
|
- newStateWordList.contains(test1));
|
|
|
- assertEquals("File state index on reload failed",
|
|
|
- 0, newStateWordList.indexOf(test1));
|
|
|
-
|
|
|
- assertTrue("File state failed to store on reload",
|
|
|
- newStateWordList.contains(test2));
|
|
|
- assertEquals("File state index on reload failed",
|
|
|
- 1, newStateWordList.indexOf(test2));
|
|
|
- } finally {
|
|
|
- lfs.delete(tempDir, true);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link FileName#FileNameState} persistence with files only.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testFileNameStateWithFiles() throws Exception {
|
|
|
- // test persistence
|
|
|
- Configuration conf = new Configuration();
|
|
|
- FileSystem lfs = FileSystem.getLocal(conf);
|
|
|
- Path rootTempDir =
|
|
|
- new Path(System.getProperty("test.build.data", "/tmp"));
|
|
|
-
|
|
|
- Path tempDir = new Path(rootTempDir, "testFileNameStateWithFiles");
|
|
|
- tempDir = lfs.makeQualified(tempDir);
|
|
|
- lfs.delete(tempDir, true);
|
|
|
-
|
|
|
- // set the persistence directory
|
|
|
- conf.set(StatePool.DIR_CONFIG, tempDir.toString());
|
|
|
- conf.setBoolean(StatePool.PERSIST_CONFIG, true);
|
|
|
-
|
|
|
- // define a state pool to help persist the filename parts
|
|
|
- StatePool pool = new StatePool();
|
|
|
-
|
|
|
- FileNameState fState = new FileNameState();
|
|
|
-
|
|
|
- // define the file names
|
|
|
- String test1 = "part-00.bzip";
|
|
|
- String test2 = "file1.txt";
|
|
|
-
|
|
|
- // test filenames only
|
|
|
- WordList fileNameState = new WordList("files");
|
|
|
- fileNameState.add(test1);
|
|
|
- fileNameState.add(test2);
|
|
|
-
|
|
|
- // set the filename state
|
|
|
- fState.setDirectoryState(fileNameState);
|
|
|
-
|
|
|
- try {
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- // add the wordlist to the pool
|
|
|
- pool.addState(getClass(), fState);
|
|
|
-
|
|
|
- // persist the state
|
|
|
- pool.persist();
|
|
|
-
|
|
|
- // now clear the pool state
|
|
|
- pool = new StatePool();
|
|
|
-
|
|
|
- // set reload to true
|
|
|
- conf.setBoolean(StatePool.RELOAD_CONFIG, true);
|
|
|
-
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- State state = pool.getState(getClass());
|
|
|
- assertNotNull("Missing state!", state);
|
|
|
- assertEquals("Incorrect state class!",
|
|
|
- FileNameState.class, state.getClass());
|
|
|
- FileNameState newFState = (FileNameState) state;
|
|
|
-
|
|
|
- // check the state contents
|
|
|
- WordList newFileWordList = newFState.getDirectoryState();
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newFileWordList.contains(test1));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 0, newFileWordList.indexOf(test1));
|
|
|
-
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newFileWordList.contains(test2));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 1, newFileWordList.indexOf(test2));
|
|
|
- } finally {
|
|
|
- lfs.delete(tempDir, true);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link FileName#FileNameState} persistence with files and directories.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testFileNameState() throws Exception {
|
|
|
- // test persistence
|
|
|
- Configuration conf = new Configuration();
|
|
|
- FileSystem lfs = FileSystem.getLocal(conf);
|
|
|
- Path rootTempDir =
|
|
|
- new Path(System.getProperty("test.build.data", "/tmp"));
|
|
|
-
|
|
|
- Path tempDir = new Path(rootTempDir, "testFileNameState");
|
|
|
- tempDir = lfs.makeQualified(tempDir);
|
|
|
- lfs.delete(tempDir, true);
|
|
|
-
|
|
|
- // set the persistence directory
|
|
|
- conf.set(StatePool.DIR_CONFIG, tempDir.toString());
|
|
|
- conf.setBoolean(StatePool.PERSIST_CONFIG, true);
|
|
|
-
|
|
|
- // define a state pool to help persist the filename parts
|
|
|
- StatePool pool = new StatePool();
|
|
|
-
|
|
|
- FileNameState fState = new FileNameState();
|
|
|
-
|
|
|
- // define the directory names
|
|
|
- String testD1 = "test";
|
|
|
- String testD2 = "home";
|
|
|
- String testD3 = "tmp";
|
|
|
-
|
|
|
- // test dir only
|
|
|
- WordList dirState = new WordList("dir");
|
|
|
- dirState.add(testD1);
|
|
|
- dirState.add(testD2);
|
|
|
- dirState.add(testD3);
|
|
|
-
|
|
|
- // define the file names
|
|
|
- String testF1 = "part-00.bzip";
|
|
|
- String testF2 = "file1.txt";
|
|
|
- String testF3 = "tmp";
|
|
|
-
|
|
|
- // test filenames only
|
|
|
- WordList fileNameState = new WordList("files");
|
|
|
- fileNameState.add(testF1);
|
|
|
- fileNameState.add(testF2);
|
|
|
- fileNameState.add(testF3);
|
|
|
-
|
|
|
- // set the filename state
|
|
|
- fState.setFileNameState(fileNameState);
|
|
|
- // set the directory state
|
|
|
- fState.setDirectoryState(dirState);
|
|
|
-
|
|
|
- try {
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- // add the wordlist to the pool
|
|
|
- pool.addState(getClass(), fState);
|
|
|
-
|
|
|
- // persist the state
|
|
|
- pool.persist();
|
|
|
-
|
|
|
- // now clear the pool state
|
|
|
- pool = new StatePool();
|
|
|
-
|
|
|
- // set reload to true
|
|
|
- conf.setBoolean(StatePool.RELOAD_CONFIG, true);
|
|
|
-
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- State state = pool.getState(getClass());
|
|
|
- assertNotNull("Missing state!", state);
|
|
|
- assertEquals("Incorrect state class!",
|
|
|
- FileNameState.class, state.getClass());
|
|
|
- FileNameState newFState = (FileNameState) state;
|
|
|
-
|
|
|
- // test filenames
|
|
|
- WordList newStateWordList = newFState.getFileNameState();
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newStateWordList.contains(testF1));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 0, newStateWordList.indexOf(testF1));
|
|
|
-
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newStateWordList.contains(testF2));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 1, newStateWordList.indexOf(testF2));
|
|
|
-
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newStateWordList.contains(testF3));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 2, newStateWordList.indexOf(testF3));
|
|
|
-
|
|
|
- // test dirs
|
|
|
- WordList newDirWordList = newFState.getDirectoryState();
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newDirWordList.contains(testD1));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 0, newDirWordList.indexOf(testD1));
|
|
|
-
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newDirWordList.contains(testD2));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 1, newDirWordList.indexOf(testD2));
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newDirWordList.contains(testD3));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 2, newDirWordList.indexOf(testD3));
|
|
|
- } finally {
|
|
|
- lfs.delete(tempDir, true);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link NodeName#NodeName} persistence with hostnames only.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testNodeNameStateWithHostNameOnly() throws Exception {
|
|
|
- // test persistence
|
|
|
- Configuration conf = new Configuration();
|
|
|
- FileSystem lfs = FileSystem.getLocal(conf);
|
|
|
- Path rootTempDir =
|
|
|
- new Path(System.getProperty("test.build.data", "/tmp"));
|
|
|
-
|
|
|
- Path tempDir = new Path(rootTempDir, "testNodeNameStateWithHostNameOnly");
|
|
|
- tempDir = lfs.makeQualified(tempDir);
|
|
|
- lfs.delete(tempDir, true);
|
|
|
-
|
|
|
- // set the persistence directory
|
|
|
- conf.set(StatePool.DIR_CONFIG, tempDir.toString());
|
|
|
- conf.setBoolean(StatePool.PERSIST_CONFIG, true);
|
|
|
-
|
|
|
- // define a state pool to help persist the hostnames
|
|
|
- StatePool pool = new StatePool();
|
|
|
-
|
|
|
- NodeNameState nState = new NodeNameState();
|
|
|
-
|
|
|
- // define the host names
|
|
|
- String test1 = "abc123";
|
|
|
- String test2 = "xyz789";
|
|
|
-
|
|
|
- // test hostname only
|
|
|
- WordList hostNameState = new WordList("hostname");
|
|
|
- hostNameState.add(test1);
|
|
|
- hostNameState.add(test2);
|
|
|
-
|
|
|
- // set the directory state
|
|
|
- nState.setHostNameState(hostNameState);
|
|
|
-
|
|
|
- try {
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- // add the wordlist to the pool
|
|
|
- pool.addState(getClass(), nState);
|
|
|
-
|
|
|
- // persist the state
|
|
|
- pool.persist();
|
|
|
-
|
|
|
- // now clear the pool state
|
|
|
- pool = new StatePool();
|
|
|
-
|
|
|
- // set reload to true
|
|
|
- conf.setBoolean(StatePool.RELOAD_CONFIG, true);
|
|
|
-
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- State state = pool.getState(getClass());
|
|
|
- assertNotNull("Missing state!", state);
|
|
|
- assertEquals("Incorrect state class!",
|
|
|
- NodeNameState.class, state.getClass());
|
|
|
- NodeNameState newNState = (NodeNameState) state;
|
|
|
-
|
|
|
- // check the state contents
|
|
|
- WordList newStateWordList = newNState.getHostNameState();
|
|
|
- assertTrue("Node state failed to store on reload",
|
|
|
- newStateWordList.contains(test1));
|
|
|
- assertEquals("Node state index on reload failed",
|
|
|
- 0, newStateWordList.indexOf(test1));
|
|
|
-
|
|
|
- assertTrue("Node state failed to store on reload",
|
|
|
- newStateWordList.contains(test2));
|
|
|
- assertEquals("Node state index on reload failed",
|
|
|
- 1, newStateWordList.indexOf(test2));
|
|
|
- } finally {
|
|
|
- lfs.delete(tempDir, true);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link NodeName#NodeNameState} persistence with racknames only.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testNodeNameWithRackNamesOnly() throws Exception {
|
|
|
- // test persistence
|
|
|
- Configuration conf = new Configuration();
|
|
|
- FileSystem lfs = FileSystem.getLocal(conf);
|
|
|
- Path rootTempDir =
|
|
|
- new Path(System.getProperty("test.build.data", "/tmp"));
|
|
|
-
|
|
|
- Path tempDir = new Path(rootTempDir, "testNodeNameWithRackNamesOnly");
|
|
|
- tempDir = lfs.makeQualified(tempDir);
|
|
|
- lfs.delete(tempDir, true);
|
|
|
-
|
|
|
- // set the persistence directory
|
|
|
- conf.set(StatePool.DIR_CONFIG, tempDir.toString());
|
|
|
- conf.setBoolean(StatePool.PERSIST_CONFIG, true);
|
|
|
-
|
|
|
- // define a state pool to help persist the rack names
|
|
|
- StatePool pool = new StatePool();
|
|
|
-
|
|
|
- NodeNameState nState = new NodeNameState();
|
|
|
-
|
|
|
- // define the rack names
|
|
|
- String test1 = "rack1";
|
|
|
- String test2 = "rack2";
|
|
|
-
|
|
|
- // test filenames only
|
|
|
- WordList rackNameState = new WordList("racknames");
|
|
|
- rackNameState.add(test1);
|
|
|
- rackNameState.add(test2);
|
|
|
-
|
|
|
- // set the rackname state
|
|
|
- nState.setRackNameState(rackNameState);
|
|
|
-
|
|
|
- try {
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- // add the wordlist to the pool
|
|
|
- pool.addState(getClass(), nState);
|
|
|
-
|
|
|
- // persist the state
|
|
|
- pool.persist();
|
|
|
-
|
|
|
- // now clear the pool state
|
|
|
- pool = new StatePool();
|
|
|
-
|
|
|
- // set reload to true
|
|
|
- conf.setBoolean(StatePool.RELOAD_CONFIG, true);
|
|
|
-
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- State state = pool.getState(getClass());
|
|
|
- assertNotNull("Missing state!", state);
|
|
|
- assertEquals("Incorrect state class!",
|
|
|
- NodeNameState.class, state.getClass());
|
|
|
- NodeNameState newNState = (NodeNameState) state;
|
|
|
-
|
|
|
- // check the state contents
|
|
|
- WordList newFileWordList = newNState.getRackNameState();
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newFileWordList.contains(test1));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 0, newFileWordList.indexOf(test1));
|
|
|
-
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newFileWordList.contains(test2));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 1, newFileWordList.indexOf(test2));
|
|
|
- } finally {
|
|
|
- lfs.delete(tempDir, true);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Test {@link NodeName#NodeNameState} persistence with hosts and racks.
|
|
|
- */
|
|
|
- @Test
|
|
|
- public void testNodeNameState() throws Exception {
|
|
|
- // test persistence
|
|
|
- Configuration conf = new Configuration();
|
|
|
- FileSystem lfs = FileSystem.getLocal(conf);
|
|
|
- Path rootTempDir =
|
|
|
- new Path(System.getProperty("test.build.data", "/tmp"));
|
|
|
-
|
|
|
- Path tempDir = new Path(rootTempDir, "testNodeNameState");
|
|
|
- tempDir = lfs.makeQualified(tempDir);
|
|
|
- lfs.delete(tempDir, true);
|
|
|
-
|
|
|
- // set the persistence directory
|
|
|
- conf.set(StatePool.DIR_CONFIG, tempDir.toString());
|
|
|
- conf.setBoolean(StatePool.PERSIST_CONFIG, true);
|
|
|
-
|
|
|
- // define a state pool to help persist the node names.
|
|
|
- StatePool pool = new StatePool();
|
|
|
-
|
|
|
- NodeNameState nState = new NodeNameState();
|
|
|
-
|
|
|
- // define the rack names
|
|
|
- String testR1 = "rack1";
|
|
|
- String testR2 = "rack2";
|
|
|
- String testR3 = "rack3";
|
|
|
-
|
|
|
- WordList rackState = new WordList("rack");
|
|
|
- rackState.add(testR1);
|
|
|
- rackState.add(testR2);
|
|
|
- rackState.add(testR3);
|
|
|
-
|
|
|
- String testH1 = "host1";
|
|
|
- String testH2 = "host2";
|
|
|
- String testH3 = "host3";
|
|
|
-
|
|
|
- WordList hostNameState = new WordList("host");
|
|
|
- hostNameState.add(testH1);
|
|
|
- hostNameState.add(testH2);
|
|
|
- hostNameState.add(testH3);
|
|
|
-
|
|
|
- // set the filename state
|
|
|
- nState.setHostNameState(hostNameState);
|
|
|
- nState.setRackNameState(rackState);
|
|
|
-
|
|
|
- try {
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- // add the wordlist to the pool
|
|
|
- pool.addState(getClass(), nState);
|
|
|
-
|
|
|
- // persist the state
|
|
|
- pool.persist();
|
|
|
-
|
|
|
- // now clear the pool state
|
|
|
- pool = new StatePool();
|
|
|
-
|
|
|
- // set reload to true
|
|
|
- conf.setBoolean(StatePool.RELOAD_CONFIG, true);
|
|
|
-
|
|
|
- // initialize the state-pool
|
|
|
- pool.initialize(conf);
|
|
|
-
|
|
|
- State state = pool.getState(getClass());
|
|
|
- assertNotNull("Missing state!", state);
|
|
|
- assertEquals("Incorrect state class!",
|
|
|
- NodeNameState.class, state.getClass());
|
|
|
- NodeNameState newNState = (NodeNameState) state;
|
|
|
-
|
|
|
- // test nodenames
|
|
|
- WordList newHostWordList = newNState.getHostNameState();
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newHostWordList.contains(testH1));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 0, newHostWordList.indexOf(testH1));
|
|
|
-
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newHostWordList.contains(testH2));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 1, newHostWordList.indexOf(testH2));
|
|
|
-
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newHostWordList.contains(testH3));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 2, newHostWordList.indexOf(testH3));
|
|
|
-
|
|
|
- // test racknames
|
|
|
- WordList newRackWordList = newNState.getRackNameState();
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newRackWordList.contains(testR1));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 0, newRackWordList.indexOf(testR1));
|
|
|
-
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newRackWordList.contains(testR2));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 1, newRackWordList.indexOf(testR2));
|
|
|
- assertTrue("File state failed on reload",
|
|
|
- newRackWordList.contains(testR3));
|
|
|
- assertEquals("File state indexing on reload failed",
|
|
|
- 2, newRackWordList.indexOf(testR3));
|
|
|
- } finally {
|
|
|
- lfs.delete(tempDir, true);
|
|
|
- }
|
|
|
- }
|
|
|
-}
|
Mahadev Konar