commit 09c2d459762c4c793b4887c48555373e7de4fbb5

Author: Vinay Kumar Thota <vinayt@yahoo-inc.com>
Date:   Tue Sep 14 04:50:31 2010 +0000

    3957917 from


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security-patches@1077676 13f79535-47bb-0310-9956-ffa450edef68

src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java

@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.test.system.MRCluster;
+import org.apache.hadoop.mapreduce.test.system.JTClient;
+import org.apache.hadoop.mapreduce.test.system.JTProtocol;
+import org.apache.hadoop.mapred.gridmix.FilePool;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+import org.junit.Test;
+import java.io.IOException;
+import java.util.ArrayList;
+
+public class TestGridMixFilePool {
+  private static final Log LOG = LogFactory
+     .getLog(TestGridMixFilePool.class);
+  private static Configuration conf = new Configuration();
+  private static MRCluster cluster;
+  private static JTProtocol remoteClient;
+  private static JTClient jtClient;
+  private static Path gridmixDir;
+  private static int clusterSize; 
+  
+  @BeforeClass
+  public static void before() throws Exception {
+    String []  excludeExpList = {"java.net.ConnectException", 
+       "java.io.IOException"};
+    cluster = MRCluster.createCluster(conf);
+    cluster.setExcludeExpList(excludeExpList);
+    cluster.setUp();
+    jtClient = cluster.getJTClient();
+    remoteClient = jtClient.getProxy();
+    clusterSize = cluster.getTTClients().size();
+    gridmixDir = new Path("hdfs:///user/" + UtilsForGridmix.getUserName()
+       + "/herriot-gridmix");
+    UtilsForGridmix.createDirs(gridmixDir, remoteClient.getDaemonConf());
+  }
+
+  @AfterClass
+  public static void after() throws Exception {
+    UtilsForGridmix.cleanup(gridmixDir, conf);
+    cluster.tearDown();
+  }
+  
+  @Test
+  public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception {
+    conf = remoteClient.getDaemonConf();
+    final long inputSize = clusterSize * 200;
+    int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500};
+    long targetSize = Long.MAX_VALUE;
+    final int expFileCount = 13;
+    String [] runtimeValues ={"LOADJOB",
+       SubmitterUserResolver.class.getName(),
+       "STRESS",
+       inputSize+"m",
+       "file:///dev/null"}; 
+
+    int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir, 
+       conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
+    Assert.assertEquals("Data generation has failed.", 0 , exitCode);
+    // create files for given sizes.
+    createFiles(new Path(gridmixDir,"input"),fileSizesInMB);
+    conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024);
+    FilePool fpool = new FilePool(conf,new Path(gridmixDir,"input"));
+    fpool.refresh();
+    verifyFilesSizeAndCountForSpecifiedPool(expFileCount,targetSize, fpool);
+  }
+  
+  private void createFiles(Path inputDir, int [] fileSizes) 
+     throws Exception {
+    for (int size : fileSizes) {
+      UtilsForGridmix.createFile(size, inputDir, conf);
+    }
+  }
+  
+  private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount, 
+     long minFileSize, FilePool pool) throws IOException {
+    final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
+    long  actFilesSize = pool.getInputFiles(minFileSize, files)/(1024 * 1024);
+    long expFilesSize = 3100 ;
+    Assert.assertEquals("Files Size has not matched for specified pool.",
+       expFilesSize, actFilesSize);
+    int actFileCount = files.size();    
+    Assert.assertEquals("File count has not matched.", 
+       expFileCount, actFileCount);
+    int count = 0;
+    for (FileStatus fstat : files) {
+      String fp = fstat.getPath().toString();
+      count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1);
+    }
+    Assert.assertEquals("Total folders are not matched with cluster size", 
+            clusterSize, count);
+  }
+}