apache
/
hadoop
mirror of https://github.com/apache/hadoop.git


			
				
					
						
						
							123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497
							<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta content="Apache Forrest" name="Generator">
<meta name="Forrest-version" content="0.8">
<meta name="Forrest-skin-name" content="pelt">
<title> Synthetic Load Generator User Guide </title>
<link type="text/css" href="skin/basic.css" rel="stylesheet">
<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
<link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
<link type="text/css" href="skin/profile.css" rel="stylesheet">
<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
<link rel="shortcut icon" href="images/favicon.ico">
</head>
<body onload="init()">
<script type="text/javascript">ndeSetTextSize();</script>
<div id="top">
<!--+
    |breadtrail
    +-->
<div class="breadtrail">
<a href="http://www.apache.org/">Apache</a> &gt; <a href="http://hadoop.apache.org/">Hadoop</a> &gt; <a href="http://hadoop.apache.org/core/">Core</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script>
</div>
<!--+
    |header
    +-->
<div class="header">
<!--+
    |start group logo
    +-->
<div class="grouplogo">
<a href="http://hadoop.apache.org/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a>
</div>
<!--+
    |end group logo
    +-->
<!--+
    |start Project Logo
    +-->
<div class="projectlogo">
<a href="http://hadoop.apache.org/core/"><img class="logoImage" alt="Hadoop" src="images/core-logo.gif" title="Scalable Computing Platform"></a>
</div>
<!--+
    |end Project Logo
    +-->
<!--+
    |start Search
    +-->
<div class="searchbox">
<form action="http://www.google.com/search" method="get" class="roundtopsmall">
<input value="hadoop.apache.org" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp; 
                    <input name="Search" value="Search" type="submit">
</form>
</div>
<!--+
    |end search
    +-->
<!--+
    |start Tabs
    +-->
<ul id="tabs">
<li>
<a class="unselected" href="http://hadoop.apache.org/core/">Project</a>
</li>
<li>
<a class="unselected" href="http://wiki.apache.org/hadoop">Wiki</a>
</li>
<li class="current">
<a class="selected" href="index.html">Hadoop 0.19 Documentation</a>
</li>
</ul>
<!--+
    |end Tabs
    +-->
</div>
</div>
<div id="main">
<div id="publishedStrip">
<!--+
    |start Subtabs
    +-->
<div id="level2tabs"></div>
<!--+
    |end Endtabs
    +-->
<script type="text/javascript"><!--
document.write("Last Published: " + document.lastModified);
//  --></script>
</div>
<!--+
    |breadtrail
    +-->
<div class="breadtrail">

             &nbsp;
           </div>
<!--+
    |start Menu, mainarea
    +-->
<!--+
    |start Menu
    +-->
<div id="menu">
<div onclick="SwitchMenu('menu_selected_1.1', 'skin/')" id="menu_selected_1.1Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Documentation</div>
<div id="menu_selected_1.1" class="selectedmenuitemgroup" style="display: block;">
<div class="menuitem">
<a href="index.html">Overview</a>
</div>
<div class="menuitem">
<a href="quickstart.html">Quickstart</a>
</div>
<div class="menuitem">
<a href="cluster_setup.html">Cluster Setup</a>
</div>
<div class="menuitem">
<a href="hdfs_design.html">HDFS Architecture</a>
</div>
<div class="menuitem">
<a href="hdfs_user_guide.html">HDFS User Guide</a>
</div>
<div class="menuitem">
<a href="hdfs_permissions_guide.html">HDFS Permissions Guide</a>
</div>
<div class="menuitem">
<a href="hdfs_quota_admin_guide.html">HDFS Quotas Administrator Guide</a>
</div>
<div class="menuitem">
<a href="commands_manual.html">Commands Manual</a>
</div>
<div class="menuitem">
<a href="hdfs_shell.html">FS Shell Guide</a>
</div>
<div class="menupage">
<div class="menupagetitle">Synthetic Load Generator User Guide</div>
</div>
<div class="menuitem">
<a href="distcp.html">DistCp Guide</a>
</div>
<div class="menuitem">
<a href="mapred_tutorial.html">Map-Reduce Tutorial</a>
</div>
<div class="menuitem">
<a href="native_libraries.html">Native Hadoop Libraries</a>
</div>
<div class="menuitem">
<a href="streaming.html">Streaming</a>
</div>
<div class="menuitem">
<a href="hadoop_archives.html">Hadoop Archives</a>
</div>
<div class="menuitem">
<a href="hod.html">Hadoop On Demand</a>
</div>
<div class="menuitem">
<a href="api/index.html">API Docs</a>
</div>
<div class="menuitem">
<a href="jdiff/changes.html">API Changes</a>
</div>
<div class="menuitem">
<a href="http://wiki.apache.org/hadoop/">Wiki</a>
</div>
<div class="menuitem">
<a href="http://wiki.apache.org/hadoop/FAQ">FAQ</a>
</div>
<div class="menuitem">
<a href="http://hadoop.apache.org/core/mailing_lists.html">Mailing Lists</a>
</div>
<div class="menuitem">
<a href="releasenotes.html">Release Notes</a>
</div>
<div class="menuitem">
<a href="changes.html">All Changes</a>
</div>
</div>
<div id="credit"></div>
<div id="roundbottom">
<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
<!--+
  |alternative credits
  +-->
<div id="credit2"></div>
</div>
<!--+
    |end Menu
    +-->
<!--+
    |start content
    +-->
<div id="content">
<div title="Portable Document Format" class="pdflink">
<a class="dida" href="SLG_user_guide.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br>
        PDF</a>
</div>
<h1> Synthetic Load Generator User Guide </h1>
<div id="minitoc-area">
<ul class="minitoc">
<li>
<a href="#Description"> Description </a>
</li>
<li>
<a href="#Synopsis"> Synopsis </a>
</li>
<li>
<a href="#Test+Space+Population"> Test Space Population </a>
<ul class="minitoc">
<li>
<a href="#Structure+Generator"> Structure Generator </a>
</li>
<li>
<a href="#Test+Space+Generator"> Test Space Generator </a>
</li>
</ul>
</li>
</ul>
</div>
		
<a name="N1000D"></a><a name="Description"></a>
<h2 class="h3"> Description </h2>
<div class="section">
<p>
        The synthetic load generator is a tool for testing NameNode behavior
        under different client loads. The user can generate different mixes 
        of read, write, and list requests by specifying the probabilities of
        read and write. The user controls the intensity of the load by adjusting
        parameters for the number of worker threads and the delay between 
        operations. While load generators are running, the user can profile and
        monitor the running of the NameNode. When a load generator exits, it
        prints some NameNode statistics like the average execution time of each
        kind of operation and the NameNode throughput.
                       </p>
</div>
		
<a name="N10017"></a><a name="Synopsis"></a>
<h2 class="h3"> Synopsis </h2>
<div class="section">
<p>
        
<span class="codefrag">java LoadGenerator [options]</span>
<br>
                        
</p>
<p>
        Options include:<br>
        
<span class="codefrag">&nbsp;&nbsp;-readProbability &lt;read probability&gt;</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the probability of the read operation;
                default is 0.3333. </span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-writeProbability &lt;write probability&gt;</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the probability of the write 
                operations; default is 0.3333.</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-root &lt;test space root&gt;</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the root of the test space;
                default is /testLoadSpace.</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-maxDelayBetweenOps 
                &lt;maxDelayBetweenOpsInMillis&gt;</span>
<br> 
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the maximum delay between two consecutive
                operations in a thread; default is 0 indicating no delay.
                </span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-numOfThreads &lt;numOfThreads&gt;</span>
<br> 
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the number of threads to spawn; 
                default is 200.</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-elapsedTime &lt;elapsedTimeInSecs&gt;</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the number of seconds that the program 
                will run; A value of zero indicates that the program runs
                forever. The default value is 0.</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-startTime &lt;startTimeInMillis&gt;</span>
<br> 
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the time that all worker threads 
                start to run. By default it is 10 seconds after the main 
                program starts running.This creates a barrier if more than
                one load generator is running.
        </span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-seed &lt;seed&gt;</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the random generator seed for repeating 
                requests to NameNode when running with a single thread;
                default is the current time.</span>
<br>
			
</p>
<p>
        After command line argument parsing, the load generator traverses 
        the test space and builds a table of all directories and another table
        of all files in the test space. It then waits until the start time to
        spawn the number of worker threads as specified by the user. Each
        thread sends a stream of requests to NameNode. At each iteration, 
        it first decides if it is going to read a file, create a file, or
        list a directory following the read and write probabilities specified
        by the user. The listing probability is equal to 
        <em>1-read probability-write probability</em>. When reading, 
        it randomly picks a file in the test space and reads the entire file. 
        When writing, it randomly picks a directory in the test space and 
        creates a file there. To avoid two threads with the same load 
        generator or from two different load generators create the same 
        file, the file name consists of the current machine's host name 
        and the thread id. The length of the file follows Gaussian 
        distribution with an average size of 2 blocks and the standard 
        deviation of 1. The new file is filled with byte 'a'. To avoid
        the test space to grow indefinitely, the file is deleted immediately
        after the file creation completes. While listing, it randomly 
        picks a directory in the test space and lists its content. 
        After an operation completes, the thread pauses for a random 
        amount of time in the range of [0, maxDelayBetweenOps] if the 
        specified maximum delay is not zero. All threads are stopped when 
        the specified elapsed time is passed. Before exiting, the program 
        prints the average execution for each kind of NameNode operations, 
        and the number of requests served by the NameNode per second.
                        </p>
</div>
                
<a name="N10070"></a><a name="Test+Space+Population"></a>
<h2 class="h3"> Test Space Population </h2>
<div class="section">
<p>
        The user needs to populate a test space before she runs a 
        load generator. The structure generator generates a random 
        test space structure and the data generator creates the files 
        and directories of the test space in Hadoop distributed file system.
                        </p>
<a name="N10079"></a><a name="Structure+Generator"></a>
<h3 class="h4"> Structure Generator </h3>
<p>
        This tool generates a random namespace structure with the 
        following constraints:
                                </p>
<ol>
        
<li>The number of subdirectories that a directory can have is 
            a random number in [minWidth, maxWidth].</li>
        
<li>The maximum depth of each subdirectory is a random number 
            [2*maxDepth/3, maxDepth].</li>
        
<li>Files are randomly placed in leaf directories. The size of 
            each file follows Gaussian distribution with an average size 
            of 1 block and a standard deviation of 1.</li>
                                        
</ol>
<p>
        The generated namespace structure is described by two files in 
        the output directory. Each line of the first file contains the 
        full name of a leaf directory. Each line of the second file 
        contains the full name of a file and its size, separated by a blank.
                                </p>
<p>
        The synopsis of the command is
                                </p>
<p>
        
<span class="codefrag">java StructureGenerator [options]</span>
                                
</p>
<p>
        Options include:<br>
        
<span class="codefrag">&nbsp;&nbsp;-maxDepth &lt;maxDepth&gt;</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;maximum depth of the directory tree; 
                default is 5.</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-minWidth &lt;minWidth&gt;</span>
<br> 
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;minimum number of subdirectories per 
                directories; default is 1.</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-maxWidth &lt;maxWidth&gt;</span>
<br> 
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;maximum number of subdirectories per 
                directories; default is 5.</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-numOfFiles &lt;#OfFiles&gt;</span>
<br> 
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the total number of files in the test 
                space; default is 10.</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-avgFileSize &lt;avgFileSizeInBlocks&gt;</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;average size of blocks; default is 1.
                </span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-outDir &lt;outDir&gt;</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;output directory; default is the 
                current directory. </span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-seed &lt;seed&gt;</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;random number generator seed; 
                default is the current time.</span>
<br>
                                
</p>
<a name="N100D8"></a><a name="Test+Space+Generator"></a>
<h3 class="h4"> Test Space Generator </h3>
<p>
        This tool reads the directory structure and file structure from 
        the input directory and creates the namespace in Hadoop distributed
        file system. All files are filled with byte 'a'.
                                </p>
<p>
        The synopsis of the command is
                                </p>
<p>
        
<span class="codefrag">java DataGenerator [options]</span>
                                
</p>
<p>
        Options include:<br>
        
<span class="codefrag">&nbsp;&nbsp;-inDir &lt;inDir&gt;</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;input directory name where directory/file
                structures are stored; default is the current directory.
        </span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;-root &lt;test space root&gt;</span>
<br>
        
<span class="codefrag">&nbsp;&nbsp;&nbsp;&nbsp;the name of the root directory which the 
                new namespace is going to be placed under; 
                default is "/testLoadSpace".</span>
<br>
                                
</p>
</div>
	
</div>
<!--+
    |end content
    +-->
<div class="clearboth">&nbsp;</div>
</div>
<div id="footer">
<!--+
    |start bottomstrip
    +-->
<div class="lastmodified">
<script type="text/javascript"><!--
document.write("Last Published: " + document.lastModified);
//  --></script>
</div>
<div class="copyright">
        Copyright &copy;
         2008 <a href="http://www.apache.org/licenses/">The Apache Software Foundation.</a>
</div>
<!--+
    |end bottomstrip
    +-->
</div>
</body>
</html>