ZOOKEEPER-498. Unending Leader Elections : WAN configuration (flavio via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/zookeeper/trunk@803300 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -62,6 +62,8 @@ BUGFIXES:
 
   ZOOKEEPER-477. zkCleanup.sh is flaky (fernando via mahadev)
 
+  ZOOKEEPER-498. Unending Leader Elections : WAN configuration (flavio via
+  mahadev)
 
 IMPROVEMENTS:
   ZOOKEEPER-473. cleanup junit tests to eliminate false positives due to

docs/bookkeeperConfig.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/bookkeeperOverview.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/bookkeeperProgrammer.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/bookkeeperStarted.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/index.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+
@@ -284,6 +284,10 @@ document.write("Last Published: " + document.lastModified);
 <li>
 <a href="zookeeperJMX.html">JMX</a> - how to enable JMX in ZooKeeper</li>
       
+<li>
+<a href="zookeeperHierarchicalQuorums.html">Hierarchical quorums</a>
+</li>
+      
 </ul>
       
 </li>

docs/index.pdf

@@ -169,10 +169,10 @@ endobj
 >>
 endobj
 20 0 obj
-<< /Length 1671 /Filter [ /ASCII85Decode /FlateDecode ]
+<< /Length 1747 /Filter [ /ASCII85Decode /FlateDecode ]
  >>
 stream
-GatU495iiK&AJ$CnCA<+W"LCK<%j/?dk/#I@Wran"o:242Iiem79>X^jPP0[#;r5L=[nO3aln)5S/FNYQ\uqL"sfk5:R9<7T558Dn_l,DW=OD3MnOX2A@X(;hY]ekqk!^[QG3DJ0q]!fpq0&6IZ*uonuBd&Ng2(dK4HTFR!MVbHr-m(?KN!<DCO%8DOskU2]h%\go5oWcj*#lb;XifV9n\5ll1F&orF<Bd<C"!aLlFSWai;#J?O\'8$8k8+I7:F4mJ(UlDNb'haR,[q<_h##\[Im*I'bq]dH+/H>+"]SfR>ac4Y.AO,Iq7qdkO72PaDj3KPF'bB[5`#)#"-(*6=TJ3BS_o$TL<;]*CVMCa8".p:qBf*)A=4\Cp()$+XPPn7N+MFt&A.Nbr2M?T8,<^"WI]](;=]Kiu7jR_%9BnJ>]"`KSP55!QNH/aVPUt(:I=h0^-S]IWgQBMp7jb9Wg4i@^W`H^\<fhojNY>Z6hUDE1Lo4)g6pNuUsSQX^u=b=f7AS2=M97q68[]aY)ofh5\40tTTil@^VV,o5u@nC%EmZi+oT?A&HIJBG0b>>F!roZ2oCj0g,i`0MNn[([2?bVejguHlq'r'G)G1*D%Rd1_WU&+1GNm$C\a=;Hc[Sj3QQ:(pKCa+8)2B0^nYD7To_T,PLL]*n>mq!frU(ZpVQYj1bStD?8p?tG)$:>t*M$q9fj[frj&`Kl"f!'e2c[f*&-$a.<N7)K^+A2MNTO(1"l'\.I"3%Cdo62fY91"QBC!eX=ZRIm3Jp_T:(paF(m%,>!^FNT=/SZlWVo?XignoU/!K1Uc2,N?mX'As_=tR[jKuOh^U7`21:WXT[#)^'N%HPKTB<dJAfU'"hJ]aHV)o`VH?4>S5f5jBR@`9uRc7lX/Zt)9N3MqULME#=EcUD9D)CRA1X-:`.`N'N!rj:,K)!BWfYZ%AISRL)7OkHkkXMq6*_J,i$7;,OpT?T3.TZHJ'3P=(Q>)6PKeHQ!WCrn,u&Ht<3H2Zg0C9_Lo2Q\d4?oA2LIQ1h<HR*W8'lP_.6X%hCl(lU\@/.D^\?IO(Bno)GSb:KE"e@'OEqq9e:p??VTSF_pl<(TTBn?h;Q]#RJR[[dAXBsY&75;c=G:-On]l[:S9;T=q:?e@`CCVdk]q7f4MS;Wh;!duA344rZhI\.+<'%LP%Ol),VoO^>dMe"f\gsf*X7J6ZDJ!a9cDY]R=TcWTh%eJ7eNt(HDbORE&r#)!^.qu-g-MuFB1941[8&]`1StFp$91eX$(YM4K4:la?HGnb.d=6TZ"lKdXZ*-82?i/.*6jo-%Q^i\/?Tbj(?3e0@g[*NKRlni)-/gE[X-)rXi%6Z$UU(Q$B5=GhZEoG/R]2t[b;n0e%KO0[95d#G*,@e&VUAgME%+BZNc<#MSnDdH&nU$mPFR#8E`DEJT@*mrAL=Jp$VUM^)eYYIcdEdk$G1nh!#&l+-\8+=Y@;;=n@]0XDFJFS+f%os&d$9P_6\dfG&rRQl/ZoAl6>'"Ff>VN(!@+*QH;o+j"d].Ja(D*GdZ2I=VoS8CGT>8NY5$@P\u57?\J]m/rNlBbE/5j24;DN>^[e9Nf`UM'(T3CpU7958&AgJ5ZFUqA/aPhRAfqo0q-<4,=Vh5H<o3,RW&m;'#7NrOV&5fP];"C@)YAALWO=\b>m1q[]LY-)(~>
+GatU48T3WI'YaHGkggH80GGg*<%il7d^3^cbY2N>5aMR_1U\3)oLO"]r;2Z2pdCOUN$9(%dC53bmr>mB/oD`/k7`N^ji[^PRt^<>p#gZcAu[Y]&W1Pk/U-&Y=+:lM^\@`tQG4u">V<`BRLd*hJ+=@=M<_8n$J!`icY!pql;uT$Go/7SHUmO=nTAi\m'+IK1Q!ah;#E<g"YWp2FN$&&ZDFA)fsLI?Jlrd5^sa"2k8jt8Ilka2YU`qq2\5o0#u7]GD%NCoNT^&u+mqYIM?Ip9WCD8uGu^ZdLIr=ONk1;T,LRb(jm'&`H)4p#KPE#=0=!9kHgrF<\JGGETN=eXNTmPjpq7E>nY_Vd$DAkAfB9ig:$Y@lMA6,=.HJ6Q<ODq6bi8]^q14?M`0t(i'9pF4hfD(+6:ESoe$>9uN^>KQ4H;a;;%OA7,RW?A-42g_Yl'kiU<%W2"U9oK6K4:K!3XR=n?S9I-.[%#pZW%'K$579T&r%<nY7XA<Ob<TbK1;5jiS]A-jrjj;4_G(l\>+;>-:8ZnQa.&3>nMr?LTGp6TgQ11r^>r7iE[ZM5nP+pAC9tkl$XXVC0s'F[eZb3:BP+0*7]Pm0fFM]nb@GF,D&>V(B>s;$1E&fU\81s)\!lDo2Mg>mc1;S"JGWZd%@1q&<*crg\Q'0AjrFlsOSt$IJ26RBW]ok.4TfWqu..%a3h981S])[t:eZQ:-U?'B,A7CfHuL"+LHjLJgg$rLilQ7)dcjiZE[e[0:JEcc[I.G<?%_Tb+kiKjp_G/-ai7G?9LWYeq_C3O!8,,]rn+plZU&A!6ut"bcP3o<Vht\.s;g_'NeeBMgfB3i8Q.KfTQp.\E4F^"FG6llK0*ITK:7Z64:2b*j#+Fiq!<&rM&Tc1'ITJ`!hTH*q=n&=XFR;_X,]&65dk3F7%\nB*[>00VFTFVoh0XITmOW7eQ.\AA57C)cVf/dPFTq"`GXVNuauN2@oA'Tl*%`D-!8.!hpW'U[/,d:LngeV?@OXY`5]@@KUN<(TDgR-gT_m+-%)imgZ;[*k<cYUBdS1X5Vi@!4+B?>Wjt`%LXr!u-ICOqT_u$4IVnI<8W"=LAX0eeqC_RmQ4c0On,m&id"NGF-o)XhJq@8Y$I0n9[lKAm'GX*>033el/&<l%K%ibARCq"eAd!$l!rOLWWmr.6,hJ0;`1+h\;htp"GuFMCDN;4c3/*KX7CV.d[[f9)%CPF%DGWrB,1#Qm1'IU#Y=Re4p?rS"4Y!l-rHhU`_jj*\Z;?CFD%*Y,F7^c<L=SRC*&VUOmc'HRL`:Y,E,<(-3+W@gY;Z8;QT@V"7nS..lf+W"uGb$)Vjb'S/uPNg!%.Am"Gnct]"N@T2bg\1CkN>)uDDN@C5hTlM:`kKIfK*E[J#=,U)eZ"]md"g]TY?`3].WuVgV4f6c\@FjF<6A/#>N<k9r.g!,f"FbILC57I;GHRLeAlUm,(R`bj;-3feTm!G=1##'h=2*RAm,'%DVsZA7P=U4("]=A6p&jB>`-QY*6!o]'Pp2&0ia(As'j#VU6/O%=MfkK[mWGu<a*h,LcaM)F?Jkc^^K$MN^3&#gK9e8KSFFdtI`LtuN+/g,WuDlfl4q0_BdQ85!4@IH#"j=BQ5=f2*m`hhS6u&'3#ViFbK&uP&n3*r.GXID75g"$a$MXpIeP2'IkT6p[XS43onbr)!iU$@"Cl%IbNAVpC#&.P$FZSf96K;!k]r8FHsOK>+TT\@e#9ZIck`H+hS^>#\bcp7*,f]~>
 endstream
 endobj
 21 0 obj
@@ -192,15 +192,16 @@ endobj
 26 0 R
 27 0 R
 28 0 R
+29 0 R
 ]
 endobj
 23 0 obj
 << /Type /Annot
 /Subtype /Link
-/Rect [ 126.0 630.4 224.964 618.4 ]
+/Rect [ 126.0 664.8 230.304 652.8 ]
 /C [ 0 0 0 ]
 /Border [ 0 0 0 ]
-/A << /URI (zookeeperInternals.html)
+/A << /URI (zookeeperHierarchicalQuorums.html)
 /S /URI >>
 /H /I
 >>
@@ -208,10 +209,10 @@ endobj
 24 0 obj
 << /Type /Annot
 /Subtype /Link
-/Rect [ 126.0 598.0 150.0 586.0 ]
+/Rect [ 126.0 611.2 224.964 599.2 ]
 /C [ 0 0 0 ]
 /Border [ 0 0 0 ]
-/A << /URI (http://wiki.apache.org/hadoop/ZooKeeper)
+/A << /URI (zookeeperInternals.html)
 /S /URI >>
 /H /I
 >>
@@ -219,10 +220,10 @@ endobj
 25 0 obj
 << /Type /Annot
 /Subtype /Link
-/Rect [ 126.0 584.8 150.0 572.8 ]
+/Rect [ 126.0 578.8 150.0 566.8 ]
 /C [ 0 0 0 ]
 /Border [ 0 0 0 ]
-/A << /URI (http://wiki.apache.org/hadoop/ZooKeeper/FAQ)
+/A << /URI (http://wiki.apache.org/hadoop/ZooKeeper)
 /S /URI >>
 /H /I
 >>
@@ -230,10 +231,10 @@ endobj
 26 0 obj
 << /Type /Annot
 /Subtype /Link
-/Rect [ 126.0 504.8 229.476 492.8 ]
+/Rect [ 126.0 565.6 150.0 553.6 ]
 /C [ 0 0 0 ]
 /Border [ 0 0 0 ]
-/A << /URI (bookkeeperOverview.html)
+/A << /URI (http://wiki.apache.org/hadoop/ZooKeeper/FAQ)
 /S /URI >>
 /H /I
 >>
@@ -241,10 +242,10 @@ endobj
 27 0 obj
 << /Type /Annot
 /Subtype /Link
-/Rect [ 126.0 491.6 257.328 479.6 ]
+/Rect [ 126.0 485.6 229.476 473.6 ]
 /C [ 0 0 0 ]
 /Border [ 0 0 0 ]
-/A << /URI (bookkeeperStarted.html)
+/A << /URI (bookkeeperOverview.html)
 /S /URI >>
 /H /I
 >>
@@ -252,43 +253,54 @@ endobj
 28 0 obj
 << /Type /Annot
 /Subtype /Link
-/Rect [ 126.0 478.4 363.972 466.4 ]
+/Rect [ 126.0 472.4 257.328 460.4 ]
 /C [ 0 0 0 ]
 /Border [ 0 0 0 ]
-/A << /URI (bookkeeperProgrammer.html)
+/A << /URI (bookkeeperStarted.html)
 /S /URI >>
 /H /I
 >>
 endobj
 29 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 126.0 459.2 363.972 447.2 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A << /URI (bookkeeperProgrammer.html)
+/S /URI >>
+/H /I
+>>
+endobj
+30 0 obj
 << /Type /Font
 /Subtype /Type1
 /Name /F3
 /BaseFont /Helvetica-Bold
 /Encoding /WinAnsiEncoding >>
 endobj
-30 0 obj
+31 0 obj
 << /Type /Font
 /Subtype /Type1
 /Name /F5
 /BaseFont /Times-Roman
 /Encoding /WinAnsiEncoding >>
 endobj
-31 0 obj
+32 0 obj
 << /Type /Font
 /Subtype /Type1
 /Name /F1
 /BaseFont /Helvetica
 /Encoding /WinAnsiEncoding >>
 endobj
-32 0 obj
+33 0 obj
 << /Type /Font
 /Subtype /Type1
 /Name /F2
 /BaseFont /Helvetica-Oblique
 /Encoding /WinAnsiEncoding >>
 endobj
-33 0 obj
+34 0 obj
 << /Type /Font
 /Subtype /Type1
 /Name /F7
@@ -307,15 +319,15 @@ endobj
 endobj
 3 0 obj
 << 
-/Font << /F3 29 0 R /F5 30 0 R /F1 31 0 R /F2 32 0 R /F7 33 0 R >> 
+/Font << /F3 30 0 R /F5 31 0 R /F1 32 0 R /F2 33 0 R /F7 34 0 R >> 
 /ProcSet [ /PDF /ImageC /Text ] >> 
 endobj
 xref
-0 34
+0 35
 0000000000 65535 f 
-0000008593 00000 n 
-0000008658 00000 n 
-0000008708 00000 n 
+0000008857 00000 n 
+0000008922 00000 n 
+0000008972 00000 n 
 0000000015 00000 n 
 0000000071 00000 n 
 0000002781 00000 n 
@@ -333,25 +345,26 @@ xref
 0000004698 00000 n 
 0000004866 00000 n 
 0000005029 00000 n 
-0000006793 00000 n 
-0000006916 00000 n 
-0000006978 00000 n 
-0000007149 00000 n 
-0000007334 00000 n 
-0000007523 00000 n 
-0000007694 00000 n 
-0000007864 00000 n 
-0000008037 00000 n 
-0000008150 00000 n 
-0000008260 00000 n 
-0000008368 00000 n 
-0000008484 00000 n 
+0000006869 00000 n 
+0000006992 00000 n 
+0000007061 00000 n 
+0000007242 00000 n 
+0000007413 00000 n 
+0000007598 00000 n 
+0000007787 00000 n 
+0000007958 00000 n 
+0000008128 00000 n 
+0000008301 00000 n 
+0000008414 00000 n 
+0000008524 00000 n 
+0000008632 00000 n 
+0000008748 00000 n 
 trailer
 <<
-/Size 34
+/Size 35
 /Root 2 0 R
 /Info 4 0 R
 >>
 startxref
-8831
+9095
 %%EOF

docs/javaExample.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/linkmap.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/recipes.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/releasenotes.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/zookeeperAdmin.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+
@@ -1150,6 +1150,9 @@ server.3=zoo3:2888:3888</span>
               The left-hand side of the assignment is a colon-separated list of server
               identifiers. Note that groups must be disjoint and the union of all groups
               must be the ZooKeeper ensemble. </p>
+<p> You will find an example <a href="zookeeperHierarchicalQuorums.html">here</a>
+              
+</p>
 </dd>
 
           
@@ -1165,11 +1168,14 @@ server.3=zoo3:2888:3888</span>
               the weight of server is 1. If the configuration defines groups, but not
               weights, then a value of 1 will be assigned to all servers.  
               </p>
+<p> You will find an example <a href="zookeeperHierarchicalQuorums.html">here</a>
+              
+</p>
 </dd>
         
 </dl>
 <p></p>
-<a name="N10383"></a><a name="sc_authOptions"></a>
+<a name="N1038F"></a><a name="sc_authOptions"></a>
 <h4>Authentication &amp; Authorization Options</h4>
 <p>The options in this section allow control over
         authentication/authorization performed by the service.</p>
@@ -1203,7 +1209,7 @@ server.3=zoo3:2888:3888</span>
 </dd>
         
 </dl>
-<a name="N103A6"></a><a name="Unsafe+Options"></a>
+<a name="N103B2"></a><a name="Unsafe+Options"></a>
 <h4>Unsafe Options</h4>
 <p>The following options can be useful, but be careful when you use
         them. The risk of each is explained along with the explanation of what
@@ -1248,7 +1254,7 @@ server.3=zoo3:2888:3888</span>
 </dd>
         
 </dl>
-<a name="N103D8"></a><a name="sc_zkCommands"></a>
+<a name="N103E4"></a><a name="sc_zkCommands"></a>
 <h3 class="h4">ZooKeeper Commands: The Four Letter Words</h3>
 <p>ZooKeeper responds to a small set of commands. Each command is
       composed of four letters. You issue the commands to ZooKeeper via telnet
@@ -1312,7 +1318,7 @@ server.3=zoo3:2888:3888</span>
 <pre class="code">$ echo ruok | nc 127.0.0.1 5111
 imok
 </pre>
-<a name="N10418"></a><a name="sc_dataFileManagement"></a>
+<a name="N10424"></a><a name="sc_dataFileManagement"></a>
 <h3 class="h4">Data File Management</h3>
 <p>ZooKeeper stores its data in a data directory and its transaction
       log in a transaction log directory. By default these two directories are
@@ -1320,7 +1326,7 @@ imok
       transaction log files in a separate directory than the data files.
       Throughput increases and latency decreases when transaction logs reside
       on a dedicated log devices.</p>
-<a name="N10421"></a><a name="The+Data+Directory"></a>
+<a name="N1042D"></a><a name="The+Data+Directory"></a>
 <h4>The Data Directory</h4>
 <p>This directory has two files in it:</p>
 <ul>
@@ -1366,14 +1372,14 @@ imok
         idempotent nature of its updates. By replaying the transaction log
         against fuzzy snapshots ZooKeeper gets the state of the system at the
         end of the log.</p>
-<a name="N1045D"></a><a name="The+Log+Directory"></a>
+<a name="N10469"></a><a name="The+Log+Directory"></a>
 <h4>The Log Directory</h4>
 <p>The Log Directory contains the ZooKeeper transaction logs.
         Before any update takes place, ZooKeeper ensures that the transaction
         that represents the update is written to non-volatile storage. A new
         log file is started each time a snapshot is begun. The log file's
         suffix is the first zxid written to that log.</p>
-<a name="N10467"></a><a name="sc_filemanagement"></a>
+<a name="N10473"></a><a name="sc_filemanagement"></a>
 <h4>File Management</h4>
 <p>The format of snapshot and log files does not change between
         standalone ZooKeeper servers and different configurations of
@@ -1393,7 +1399,7 @@ imok
         this document for more details on setting a retention policy
         and maintenance of ZooKeeper storage.
         </p>
-<a name="N1047C"></a><a name="sc_commonProblems"></a>
+<a name="N10488"></a><a name="sc_commonProblems"></a>
 <h3 class="h4">Things to Avoid</h3>
 <p>Here are some common problems you can avoid by configuring
       ZooKeeper correctly:</p>
@@ -1447,7 +1453,7 @@ imok
 </dd>
       
 </dl>
-<a name="N104A0"></a><a name="sc_bestPractices"></a>
+<a name="N104AC"></a><a name="sc_bestPractices"></a>
 <h3 class="h4">Best Practices</h3>
 <p>For best results, take note of the following list of good
       Zookeeper practices:</p>

docs/zookeeperHierarchicalQuorums.html

@@ -0,0 +1,272 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<meta content="Apache Forrest" name="Generator">
+<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-skin-name" content="pelt">
+<title>Introduction to hierarchical quorums</title>
+<link type="text/css" href="skin/basic.css" rel="stylesheet">
+<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
+<link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
+<link type="text/css" href="skin/profile.css" rel="stylesheet">
+<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
+<link rel="shortcut icon" href="images/favicon.ico">
+</head>
+<body onload="init()">
+<script type="text/javascript">ndeSetTextSize();</script>
+<div id="top">
+<!--+
+    |breadtrail
+    +-->
+<div class="breadtrail">
+<a href="http://www.apache.org/">Apache</a> &gt; <a href="http://hadoop.apache.org/">Hadoop</a> &gt; <a href="http://hadoop.apache.org/zookeeper/">ZooKeeper</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script>
+</div>
+<!--+
+    |header
+    +-->
+<div class="header">
+<!--+
+    |start group logo
+    +-->
+<div class="grouplogo">
+<a href="http://hadoop.apache.org/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a>
+</div>
+<!--+
+    |end group logo
+    +-->
+<!--+
+    |start Project Logo
+    +-->
+<div class="projectlogo">
+<a href="http://hadoop.apache.org/zookeeper/"><img class="logoImage" alt="ZooKeeper" src="images/zookeeper_small.gif" title="ZooKeeper: distributed coordination"></a>
+</div>
+<!--+
+    |end Project Logo
+    +-->
+<!--+
+    |start Search
+    +-->
+<div class="searchbox">
+<form action="http://www.google.com/search" method="get" class="roundtopsmall">
+<input value="hadoop.apache.org" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp; 
+                    <input name="Search" value="Search" type="submit">
+</form>
+</div>
+<!--+
+    |end search
+    +-->
+<!--+
+    |start Tabs
+    +-->
+<ul id="tabs">
+<li>
+<a class="unselected" href="http://hadoop.apache.org/zookeeper/">Project</a>
+</li>
+<li>
+<a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
+</li>
+<li class="current">
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
+</li>
+</ul>
+<!--+
+    |end Tabs
+    +-->
+</div>
+</div>
+<div id="main">
+<div id="publishedStrip">
+<!--+
+    |start Subtabs
+    +-->
+<div id="level2tabs"></div>
+<!--+
+    |end Endtabs
+    +-->
+<script type="text/javascript"><!--
+document.write("Last Published: " + document.lastModified);
+//  --></script>
+</div>
+<!--+
+    |breadtrail
+    +-->
+<div class="breadtrail">
+
+             &nbsp;
+           </div>
+<!--+
+    |start Menu, mainarea
+    +-->
+<!--+
+    |start Menu
+    +-->
+<div id="menu">
+<div onclick="SwitchMenu('menu_1.1', 'skin/')" id="menu_1.1Title" class="menutitle">Overview</div>
+<div id="menu_1.1" class="menuitemgroup">
+<div class="menuitem">
+<a href="index.html">Welcome</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperOver.html">Overview</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperStarted.html">Getting Started</a>
+</div>
+<div class="menuitem">
+<a href="releasenotes.html">Release Notes</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Developer</div>
+<div id="menu_1.2" class="menuitemgroup">
+<div class="menuitem">
+<a href="api/index.html">API Docs</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperProgrammers.html">Programmer's Guide</a>
+</div>
+<div class="menuitem">
+<a href="javaExample.html">Java Example</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperTutorial.html">Barrier and Queue Tutorial</a>
+</div>
+<div class="menuitem">
+<a href="recipes.html">Recipes</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">BookKeeper</div>
+<div id="menu_1.3" class="menuitemgroup">
+<div class="menuitem">
+<a href="bookkeeperStarted.html">Getting started</a>
+</div>
+<div class="menuitem">
+<a href="bookkeeperOverview.html">Overview</a>
+</div>
+<div class="menuitem">
+<a href="bookkeeperConfig.html">Setup guide</a>
+</div>
+<div class="menuitem">
+<a href="bookkeeperProgrammer.html">Programmer's guide</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_1.4', 'skin/')" id="menu_1.4Title" class="menutitle">Admin &amp; Ops</div>
+<div id="menu_1.4" class="menuitemgroup">
+<div class="menuitem">
+<a href="zookeeperAdmin.html">Administrator's Guide</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperQuotas.html">Quota Guide</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperJMX.html">JMX</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_1.5', 'skin/')" id="menu_1.5Title" class="menutitle">Contributor</div>
+<div id="menu_1.5" class="menuitemgroup">
+<div class="menuitem">
+<a href="zookeeperInternals.html">ZooKeeper Internals</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_1.6', 'skin/')" id="menu_1.6Title" class="menutitle">Miscellaneous</div>
+<div id="menu_1.6" class="menuitemgroup">
+<div class="menuitem">
+<a href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
+</div>
+<div class="menuitem">
+<a href="http://wiki.apache.org/hadoop/ZooKeeper/FAQ">FAQ</a>
+</div>
+<div class="menuitem">
+<a href="http://hadoop.apache.org/zookeeper/mailing_lists.html">Mailing Lists</a>
+</div>
+</div>
+<div id="credit"></div>
+<div id="roundbottom">
+<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
+<!--+
+  |alternative credits
+  +-->
+<div id="credit2"></div>
+</div>
+<!--+
+    |end Menu
+    +-->
+<!--+
+    |start content
+    +-->
+<div id="content">
+<div title="Portable Document Format" class="pdflink">
+<a class="dida" href="zookeeperHierarchicalQuorums.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br>
+        PDF</a>
+</div>
+<h1>Introduction to hierarchical quorums</h1>
+  
+
+  
+
+    
+<p>
+    This document gives an example of how to use hierarchical quorums. The basic idea is
+    very simple. First, we split servers into groups, and add a line for each group listing
+    the servers that form this group. Next we have to assign a weight to each server.  
+    </p>
+    
+    
+<p>
+    The following example shows how to configure a system with three groups of three servers
+    each, and we assign a weight of 1 to each server:
+    </p>
+    
+    
+<pre class="code">
+    group.1=1:2:3
+    group.2=4:5:6
+    group.3=7:8:9
+   
+    weight.1=1
+    weight.2=1
+    weight.3=1
+    weight.4=1
+    weight.5=1
+    weight.6=1
+    weight.7=1
+    weight.8=1
+    weight.9=1
+ 	</pre>
+
+	
+<p>    
+    When running the system, we are able to form a quorum once we have a majority of votes from
+    a majority of non-zero-weight groups. Groups that have zero weight are discarded and not
+    considered when forming quorums. Looking at the example, we are able to form a quorum once
+    we have votes from at least two servers from each of two different groups.
+    </p> 
+ 
+<p align="right">
+<font size="-2"></font>
+</p>
+</div>
+<!--+
+    |end content
+    +-->
+<div class="clearboth">&nbsp;</div>
+</div>
+<div id="footer">
+<!--+
+    |start bottomstrip
+    +-->
+<div class="lastmodified">
+<script type="text/javascript"><!--
+document.write("Last Published: " + document.lastModified);
+//  --></script>
+</div>
+<div class="copyright">
+        Copyright &copy;
+         2008 <a href="http://www.apache.org/licenses/">The Apache Software Foundation.</a>
+</div>
+<!--+
+    |end bottomstrip
+    +-->
+</div>
+</body>
+</html>

docs/zookeeperHierarchicalQuorums.pdf

@@ -0,0 +1,94 @@
+%PDF-1.3
+%ª«¬­
+4 0 obj
+<< /Type /Info
+/Producer (FOP 0.20.5) >>
+endobj
+5 0 obj
+<< /Length 1156 /Filter [ /ASCII85Decode /FlateDecode ]
+ >>
+stream
+Gatm;d>jt['Re;/pi0?gEG]2>7%/mKeQK;[.^b8cLUI^%R,F:gX::AarVDr_iL2Kr[]8m0Y&j+*3PI]/q/Fu7ob\jl0gUS07u"3V8'l6G&TJnMZ=,<C"@l%@%O^o'h03^KFP.J$dd9,<cU@%0GF`'3CF)\mMHUa?Z]'@u[HUK\*VojY^;$Pi`iNLH3/MSe9rX^jDWhp7(4pUY\]@88TiQ?X(D":9Q+H0TZL*s?UO&4+OLNE>r1'D)NEc[0[63(;Z32E"n[Z-eNA^$AFCL5rO[BRaHHN?>Q;Ei%#8())ZS]!*^&SbcC_jgdJHX'eW/U-#0C]4A[c^*\ENCuL#KWiI8G-Y<J%"g>H6ihh(bC`-C-fj3)k_F^e>qimHaZl?&Z?1.].Gl_#G[[1EccH\H.bu46E'<04j`Rdq7-m6<p.da>R\0NlIZ;P\4CbD]C9UM%O3hNU;jMJVb="9\i,P^G^@''BBtEl&Deu&6FW4@p-o7CfGE:6T]/C_2:U,H\IoY<KL]b\I:GcmRc%_R4c$LGK5d1V`Ep>[d=8l!qe<1i*d40GD^J_N(_J<4,hVT<g^$7/Lphg2q'4UQ'V.Oe,"DL!i1;u*bE.kMC%p*E_r/lZU;SuT8dEQ%$,qM8UGcTkdg*Hsqe2he"HP!,'+`A0pcdO^@5rIk90@Ik+SP%`$c<k"/=k^ob/7u2=D-cdq,k(Z,n.hNO,Z'%;9#g_5*g+q;he6=A)f/B?oVA19=D(-K@(DmZ%\/^bZ<NI/M)r:?ZF8,fZamD;1[d'r;9_kE%4Q=[Vr\BorN'gO5)OIVe3g[IEr(pE,:Hj'Xi](BCZKJ9QA5Te4+R6iM6=P3uR[SLi=?YH;m@hfD#""X`!UB8>GY*AhmO2BI"[$*pC'YTA)'[[SM1+Qc^)S.4+>lK,;8!b;+t>9XiP[6Ke!lh)>b4jPn_1Y^9])]Y;VfT_2Y5"OcnaC$J7hF$0[4$g4*;iW!(n;t\qoqIK^V2DVb'nHCK;eW%_D`pYB`F+glaWiFi$[@o&P*QVO>YOj7Upsr%CG!=t*8MrF>G&&?+VbB)E>Aj9oNcF3o.#XU&N:MClcS&l__snNB4,`>90ga;7RP#mrnFbK)"7\%Ul7UODO!B^sDp*P')hGh@%J\+cc##`tYF:c-qVSQ%#.F~>
+endstream
+endobj
+6 0 obj
+<< /Type /Page
+/Parent 1 0 R
+/MediaBox [ 0 0 612 792 ]
+/Resources 3 0 R
+/Contents 5 0 R
+>>
+endobj
+7 0 obj
+<< /Type /Font
+/Subtype /Type1
+/Name /F3
+/BaseFont /Helvetica-Bold
+/Encoding /WinAnsiEncoding >>
+endobj
+8 0 obj
+<< /Type /Font
+/Subtype /Type1
+/Name /F5
+/BaseFont /Times-Roman
+/Encoding /WinAnsiEncoding >>
+endobj
+9 0 obj
+<< /Type /Font
+/Subtype /Type1
+/Name /F1
+/BaseFont /Helvetica
+/Encoding /WinAnsiEncoding >>
+endobj
+10 0 obj
+<< /Type /Font
+/Subtype /Type1
+/Name /F9
+/BaseFont /Courier
+/Encoding /WinAnsiEncoding >>
+endobj
+11 0 obj
+<< /Type /Font
+/Subtype /Type1
+/Name /F2
+/BaseFont /Helvetica-Oblique
+/Encoding /WinAnsiEncoding >>
+endobj
+1 0 obj
+<< /Type /Pages
+/Count 1
+/Kids [6 0 R ] >>
+endobj
+2 0 obj
+<< /Type /Catalog
+/Pages 1 0 R
+ >>
+endobj
+3 0 obj
+<< 
+/Font << /F3 7 0 R /F5 8 0 R /F1 9 0 R /F9 10 0 R /F2 11 0 R >> 
+/ProcSet [ /PDF /ImageC /Text ] >> 
+endobj
+xref
+0 12
+0000000000 65535 f 
+0000001975 00000 n 
+0000002033 00000 n 
+0000002083 00000 n 
+0000000015 00000 n 
+0000000071 00000 n 
+0000001319 00000 n 
+0000001425 00000 n 
+0000001537 00000 n 
+0000001646 00000 n 
+0000001753 00000 n 
+0000001859 00000 n 
+trailer
+<<
+/Size 12
+/Root 2 0 R
+/Info 4 0 R
+>>
+startxref
+2203
+%%EOF

docs/zookeeperInternals.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/zookeeperJMX.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/zookeeperOver.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/zookeeperProgrammers.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/zookeeperQuotas.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/zookeeperStarted.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

docs/zookeeperTutorial.html

@@ -67,7 +67,7 @@
 <a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
 </li>
 <li class="current">
-<a class="selected" href="index.html">ZooKeeper 3.2 Documentation</a>
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
 </li>
 </ul>
 <!--+

src/docs/src/documentation/content/xdocs/index.xml

@@ -63,6 +63,7 @@
       <li><a href="zookeeperAdmin.html">Administrator's Guide</a> - a guide for system administrators and anyone else who might deploy ZooKeeper</li>
       <li><a href="zookeeperQuotas.html">Quota Guide</a> - a guide for system administrators on Quotas in ZooKeeper. </li>
       <li><a href="zookeeperJMX.html">JMX</a> - how to enable JMX in ZooKeeper</li>
+      <li><a href="zookeeperHierarchicalQuorums.html">Hierarchical quorums</a></li>
       </ul>
       </li>
       

src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml

@@ -800,6 +800,9 @@ server.3=zoo3:2888:3888</computeroutput></para>
               The left-hand side of the assignment is a colon-separated list of server
               identifiers. Note that groups must be disjoint and the union of all groups
               must be the ZooKeeper ensemble. </para>
+              
+              <para> You will find an example <ulink url="zookeeperHierarchicalQuorums.html">here</ulink>
+              </para>
             </listitem>
           </varlistentry>
 
@@ -816,6 +819,9 @@ server.3=zoo3:2888:3888</computeroutput></para>
               the weight of server is 1. If the configuration defines groups, but not
               weights, then a value of 1 will be assigned to all servers.  
               </para>
+              
+              <para> You will find an example <ulink url="zookeeperHierarchicalQuorums.html">here</ulink>
+              </para>
             </listitem>
           </varlistentry>
         </variablelist>

src/docs/src/documentation/content/xdocs/zookeeperHierarchicalQuorums.xml

@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Copyright 2002-2004 The Apache Software Foundation
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!DOCTYPE article PUBLIC "-//OASIS//DTD Simplified DocBook XML V1.0//EN"
+"http://www.oasis-open.org/docbook/xml/simple/1.0/sdocbook.dtd">
+<article id="zk_HierarchicalQuorums">
+  <title>Introduction to hierarchical quorums</title>
+
+  <articleinfo>
+    <legalnotice>
+      <para>Licensed under the Apache License, Version 2.0 (the "License");
+      you may not use this file except in compliance with the License. You may
+      obtain a copy of the License at <ulink
+      url="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</ulink>.</para>
+
+      <para>Unless required by applicable law or agreed to in writing,
+      software distributed under the License is distributed on an "AS IS"
+      BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied. See the License for the specific language governing permissions
+      and limitations under the License.</para>
+    </legalnotice>
+
+    <abstract>
+      <para>This document contains information about hierarchical quorums.</para>
+    </abstract>
+  </articleinfo>
+
+    <para>
+    This document gives an example of how to use hierarchical quorums. The basic idea is
+    very simple. First, we split servers into groups, and add a line for each group listing
+    the servers that form this group. Next we have to assign a weight to each server.  
+    </para>
+    
+    <para>
+    The following example shows how to configure a system with three groups of three servers
+    each, and we assign a weight of 1 to each server:
+    </para>
+    
+    <programlisting>
+    group.1=1:2:3
+    group.2=4:5:6
+    group.3=7:8:9
+   
+    weight.1=1
+    weight.2=1
+    weight.3=1
+    weight.4=1
+    weight.5=1
+    weight.6=1
+    weight.7=1
+    weight.8=1
+    weight.9=1
+ 	</programlisting>
+
+	<para>    
+    When running the system, we are able to form a quorum once we have a majority of votes from
+    a majority of non-zero-weight groups. Groups that have zero weight are discarded and not
+    considered when forming quorums. Looking at the example, we are able to form a quorum once
+    we have votes from at least two servers from each of two different groups.
+    </para> 
+ </article>

src/java/main/org/apache/zookeeper/server/quorum/FastLeaderElection.java

@@ -172,20 +172,22 @@ public class FastLeaderElection implements Election {
          */
         
         class WorkerReceiver implements Runnable {
-
+            volatile boolean stop;
         	QuorumCnxManager manager;
 
             WorkerReceiver(QuorumCnxManager manager) {
+                this.stop = false;
                 this.manager = manager;
             }
 
             public void run() {
                 
             	Message response;
-            	while (true) {
+            	while (!stop) {
                     // Sleeps on receive
             		try{
-            			response = manager.recvQueue.take();
+            			response = manager.recvQueue.poll(3000, TimeUnit.MILLISECONDS);
+            			if(response == null) continue;
             			
             			// Receive new message
             			LOG.debug("Receive new message.");
@@ -266,6 +268,7 @@ public class FastLeaderElection implements Election {
             					e.toString());
             		}
             	}
+                LOG.info("WorkerReceiver is down");
             }
         }
 
@@ -276,23 +279,26 @@ public class FastLeaderElection implements Election {
          */
         
         class WorkerSender implements Runnable {
-        	
+        	volatile boolean stop;
             QuorumCnxManager manager;
 
             WorkerSender(QuorumCnxManager manager){ 
+                this.stop = false;
                 this.manager = manager;
             }
             
             public void run() {
-                while (true) {
+                while (!stop) {
                     try {
-                        ToSend m = sendqueue.take();
+                        ToSend m = sendqueue.poll(3000, TimeUnit.MILLISECONDS);
+                        if(m == null) continue;
+                        
                         process(m);
                     } catch (InterruptedException e) {
                         break;
                     }
-
                 }
+                LOG.info("WorkerSender is down");
             }
 
             /**
@@ -326,6 +332,10 @@ public class FastLeaderElection implements Election {
             return (sendqueue.isEmpty() || recvqueue.isEmpty());
         }
 
+        
+        WorkerSender ws;
+        WorkerReceiver wr;
+        
         /**
          * Constructor of class Messenger.
          * 
@@ -333,20 +343,33 @@ public class FastLeaderElection implements Election {
          */
         Messenger(QuorumCnxManager manager) {
 
-            Thread t = new Thread(new WorkerSender(manager),
+            this.ws = new WorkerSender(manager);
+            
+            Thread t = new Thread(this.ws,
             		"WorkerSender Thread");
             t.setDaemon(true);
             t.start();
 
-            t = new Thread(new WorkerReceiver(manager),
+            this.wr = new WorkerReceiver(manager);
+        
+            t = new Thread(this.wr,
                     				"WorkerReceiver Thread");
             t.setDaemon(true);
             t.start();
         }
+        
+        /**
+         * Stops instances of WorkerSender and WorkerReceiver
+         */
+        void halt(){
+            this.ws.stop = true;
+            this.wr.stop = true;
+        }
 
     }
 
-    QuorumPeer self;
+    QuorumPeer self;    
+    Messenger messenger;
     volatile long logicalclock; /* Election instance */
     long proposedLeader;
     long proposedZxid;
@@ -369,7 +392,8 @@ public class FastLeaderElection implements Election {
      * @param manager   Connection manager
      */
     public FastLeaderElection(QuorumPeer self, QuorumCnxManager manager){
-    	this.manager = manager;
+    	this.stop = false;
+        this.manager = manager;
     	starter(self, manager);
     }
     
@@ -390,7 +414,7 @@ public class FastLeaderElection implements Election {
 
         sendqueue = new LinkedBlockingQueue<ToSend>();
         recvqueue = new LinkedBlockingQueue<Notification>();
-        new Messenger(manager);
+        this.messenger = new Messenger(manager);
     }
 
     private void leaveInstance() {
@@ -401,9 +425,14 @@ public class FastLeaderElection implements Election {
     	return manager;
     }
     
+    volatile boolean stop;
     public void shutdown(){
+        stop = true;
         LOG.debug("Shutting down connection manager");
         manager.halt();
+        LOG.debug("Shutting down messenger");
+        messenger.halt();
+        LOG.debug("FLE is down");
     }
 
     
@@ -434,6 +463,10 @@ public class FastLeaderElection implements Election {
      */
     private boolean totalOrderPredicate(long newId, long newZxid, long curId, long curZxid) {
         LOG.debug("id: " + newId + ", proposed id: " + curId + ", zxid: " + newZxid + ", proposed zxid: " + curZxid);
+        if(self.getQuorumVerifier().getWeight(newId) == 0){
+            return false;
+        }
+        
         if ((newZxid > curZxid)
                 || ((newZxid == curZxid) && (newId > curId)))
             return true;
@@ -550,7 +583,8 @@ public class FastLeaderElection implements Election {
              * Loop in which we exchange notifications until we find a leader
              */
     
-            while (self.getPeerState() == ServerState.LOOKING) {
+            while ((self.getPeerState() == ServerState.LOOKING) &&
+                    (!stop)){
                 /*
                  * Remove next notification from queue, times out after 2 times
                  * the termination time
@@ -606,14 +640,12 @@ public class FastLeaderElection implements Election {
                         }
                     
                         LOG.info("Adding vote");
-                        /*
-                         * Skip zero-weight servers
-                         */
-                        if(self.getQuorumVerifier().getWeight(n.sid) != 0) 
-                            recvset.put(n.sid, new Vote(n.leader, n.zxid, n.epoch));
+
+                        recvset.put(n.sid, new Vote(n.leader, n.zxid, n.epoch));
     
                         //If have received from all nodes, then terminate
-                        if (self.quorumPeers.size() == recvset.size()) {
+                        if ((self.quorumPeers.size() == recvset.size()) &&
+                                (self.getQuorumVerifier().getWeight(proposedLeader) != 0)){
                             self.setPeerState((proposedLeader == self.getId()) ? 
                                     ServerState.LEADING: ServerState.FOLLOWING);
                             leaveInstance();

src/java/main/org/apache/zookeeper/server/quorum/QuorumCnxManager.java

@@ -338,7 +338,7 @@ public class QuorumCnxManager {
                         e);
             }
         } else {
-            LOG.error("There is a connection for server " + sid);
+            LOG.debug("There is a connection already for server " + sid);
         }
     }
     

src/java/main/org/apache/zookeeper/server/quorum/flexible/QuorumHierarchical.java

@@ -121,8 +121,8 @@ public class QuorumHierarchical implements QuorumVerifier {
         this.serverGroup = serverGroup;
         this.groupWeight = new HashMap<Long, Long>();
         
-        computeGroupWeight();
         this.numGroups = numGroups;
+        computeGroupWeight();   
     }
     
     
@@ -219,7 +219,11 @@ public class QuorumHierarchical implements QuorumVerifier {
          * Do not consider groups with weight zero
          */
         for(long weight: groupWeight.values()){
-            if(weight == 0) numGroups--;
+            LOG.debug("Group weight: " + weight);
+            if(weight == ((long) 0)){
+                numGroups--;
+                LOG.debug("One zero-weight group: " + 1 + ", " + numGroups);
+            }
         }
     }
     
@@ -233,7 +237,7 @@ public class QuorumHierarchical implements QuorumVerifier {
          * Adds up weights per group
          */
         if(set.size() == 0) return false;
-        else LOG.info("Set size: " + set.size());
+        else LOG.debug("Set size: " + set.size());
         
         for(long sid : set){
             Long gid = serverGroup.get(sid);
@@ -250,17 +254,18 @@ public class QuorumHierarchical implements QuorumVerifier {
          */
         int majGroupCounter = 0;
         for(long gid : expansion.keySet()) {
-            LOG.info("gid: " + expansion.get(gid));
+            LOG.debug("Group info: " + expansion.get(gid) + ", " + gid + ", " + groupWeight.get(gid));
             if(expansion.get(gid) > (groupWeight.get(gid) / 2) )
                 majGroupCounter++;
         }
         
+        LOG.debug("Majority group counter: " + majGroupCounter + ", " + numGroups); 
         if((majGroupCounter > (numGroups / 2))){
-            LOG.info("Positive set size: " + set.size());
+            LOG.debug("Positive set size: " + set.size());
             return true;
         }
         else {
-            LOG.info("Negative set size: " + set.size());
+            LOG.debug("Negative set size: " + set.size());
             return false;
         }
     }

src/java/test/org/apache/zookeeper/test/FLEZeroWeightTest.java

@@ -103,7 +103,7 @@ public class FLEZeroWeightTest extends TestCase {
     class LEThread extends Thread {
         int i;
         QuorumPeer peer;
-        //int peerRound = 1;
+        boolean fail;
 
         LEThread(QuorumPeer peer, int i) {
             this.i = i;
@@ -114,6 +114,7 @@ public class FLEZeroWeightTest extends TestCase {
         public void run() {
             try {
                 Vote v = null;
+                fail = false;
                 while(true){
 
                     //while(true) {
@@ -135,7 +136,7 @@ public class FLEZeroWeightTest extends TestCase {
                     votes[i] = v;
 
                     if((peer.getPeerState() == ServerState.LEADING) &&
-                            (peer.getId() > 2)) fail("Elected zero-weight server");
+                            (peer.getId() > 2)) fail = true;
                     
                     if((peer.getPeerState() == ServerState.FOLLOWING) ||
                             (peer.getPeerState() == ServerState.LEADING)) break;
@@ -148,10 +149,10 @@ public class FLEZeroWeightTest extends TestCase {
     }
 
     @Test
-    public void testHierarchicalQuorum() throws Exception {
+    public void testZeroWeightQuorum() throws Exception {
         FastLeaderElection le[] = new FastLeaderElection[count];
 
-        LOG.info("TestHierarchicalQuorum: " + getName()+ ", " + count);
+        LOG.info("TestZeroWeightQuorum: " + getName()+ ", " + count);
         for(int i = 0; i < count; i++) {
             peers.put(Long.valueOf(i),
                     new QuorumServer(i,
@@ -175,6 +176,9 @@ public class FLEZeroWeightTest extends TestCase {
             threads.get(i).join(15000);
             if (threads.get(i).isAlive()) {
                 fail("Threads didn't join");
+            } else {
+                if(threads.get(i).fail)
+                    fail("Elected zero-weight server");
             }
         }
     }

src/java/test/org/apache/zookeeper/test/HierarchicalQuorumTest.java

@@ -1,5 +1,4 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
+/* Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
@@ -17,172 +16,257 @@
  */
 
 package org.apache.zookeeper.test;
+
 import java.io.ByteArrayInputStream;
 import java.io.File;
+import java.io.IOException;
 import java.net.InetSocketAddress;
-import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashSet;
 import java.util.Properties;
-import java.util.Random;
-
-import junit.framework.TestCase;
+import java.util.Set;
 
 import org.apache.log4j.Logger;
 import org.apache.zookeeper.PortAssignment;
+import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.server.quorum.FastLeaderElection;
 import org.apache.zookeeper.server.quorum.QuorumPeer;
-import org.apache.zookeeper.server.quorum.Vote;
 import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
-import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState;
 import org.apache.zookeeper.server.quorum.flexible.QuorumHierarchical;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-public class HierarchicalQuorumTest extends TestCase {
-    private static final Logger LOG = Logger.getLogger(HierarchicalQuorumTest.class);
-
-    Properties qp;
+/**
+ * Comprehensive test of hierarchical quorums, assuming servers with zero weight.
+ * This test uses ClientTest to verify that the ensemble works after a leader is 
+ * elected.
+ * 
+ * This implementation is based on QuorumBase, the main difference being that it 
+ * uses hierarchical quorums and FLE.
+ */
 
-    int count;
-    HashMap<Long,QuorumServer> peers;
-    ArrayList<LEThread> threads;
-    File tmpdir[];
-    int port[];
-    Object finalObj;
+public class HierarchicalQuorumTest extends ClientBase {
+    private static final Logger LOG = Logger.getLogger(QuorumBase.class);
 
-    volatile Vote votes[];
-    volatile boolean leaderDies;
-    volatile long leader = -1;
-    Random rand = new Random();
+    File s1dir, s2dir, s3dir, s4dir, s5dir;
+    QuorumPeer s1, s2, s3, s4, s5;
+    private int port1;
+    private int port2;
+    private int port3;
+    private int port4;
+    private int port5;
 
+    private int leport1;
+    private int leport2;
+    private int leport3;
+    private int leport4;
+    private int leport5;
 
-    @Before
+    Properties qp;
+    private final ClientTest ct = new ClientTest();
+    
     @Override
     protected void setUp() throws Exception {
-        count = 9;
-
-        peers = new HashMap<Long,QuorumServer>(count);
-        threads = new ArrayList<LEThread>(count);
-        votes = new Vote[count];
-        tmpdir = new File[count];
-        port = new int[count];
-        finalObj = new Object();
-
-        String config = "group.1=0:1:2\n" +
-        "group.2=3:4:5\n" +
-        "group.3=6:7:8\n\n" +
-        "weight.0=1\n" +
+        LOG.info("STARTING " + getName());
+        setupTestEnv();
+
+        JMXEnv.setUp();
+
+        setUpAll();
+
+        port1 = PortAssignment.unique();
+        port2 = PortAssignment.unique();
+        port3 = PortAssignment.unique();
+        port4 = PortAssignment.unique();
+        port5 = PortAssignment.unique();
+        leport1 = PortAssignment.unique();
+        leport2 = PortAssignment.unique();
+        leport3 = PortAssignment.unique();
+        leport4 = PortAssignment.unique();
+        leport5 = PortAssignment.unique();
+
+        hostPort = "127.0.0.1:" + port1 
+            + ",127.0.0.1:" + port2 
+            + ",127.0.0.1:" + port3 
+            + ",127.0.0.1:" + port4 
+            + ",127.0.0.1:" + port5;
+        LOG.info("Ports are: " + hostPort);
+
+        s1dir = ClientBase.createTmpDir();
+        s2dir = ClientBase.createTmpDir();
+        s3dir = ClientBase.createTmpDir();
+        s4dir = ClientBase.createTmpDir();
+        s5dir = ClientBase.createTmpDir();
+
+        String config = "group.1=1:2:3\n" +
+        "group.2=4:5\n" +
         "weight.1=1\n" +
         "weight.2=1\n" +
         "weight.3=1\n" +
-        "weight.4=1\n" +
-        "weight.5=1\n" +
-        "weight.6=1\n" +
-        "weight.7=1\n" +
-        "weight.8=1";
+        "weight.4=0\n" +
+        "weight.5=0\n";
 
         ByteArrayInputStream is = new ByteArrayInputStream(config.getBytes());
         this.qp = new Properties();
+        
         qp.load(is);
+        startServers();
 
-        LOG.info("SetUp " + getName());
+        ct.hostPort = hostPort;
+        ct.setUpAll();
+        
+        LOG.info("Setup finished");
     }
+    
+    
+    void startServers() throws Exception {
+        int tickTime = 2000;
+        int initLimit = 3;
+        int syncLimit = 3;
+        HashMap<Long,QuorumServer> peers = new HashMap<Long,QuorumServer>();
+        peers.put(Long.valueOf(1), new QuorumServer(1, 
+                new InetSocketAddress("127.0.0.1", port1 + 1000),
+                new InetSocketAddress("127.0.0.1", leport1 + 1000)));
+        peers.put(Long.valueOf(2), new QuorumServer(2, 
+                new InetSocketAddress("127.0.0.1", port2 + 1000),
+                new InetSocketAddress("127.0.0.1", leport2 + 1000)));
+        peers.put(Long.valueOf(3), new QuorumServer(3, 
+                new InetSocketAddress("127.0.0.1", port3 + 1000),
+                new InetSocketAddress("127.0.0.1", leport3 + 1000)));
+        peers.put(Long.valueOf(4), new QuorumServer(4,
+                new InetSocketAddress("127.0.0.1", port4 + 1000),
+                new InetSocketAddress("127.0.0.1", leport4 + 1000)));
+        peers.put(Long.valueOf(5), new QuorumServer(5,
+                new InetSocketAddress("127.0.0.1", port5 + 1000),
+                new InetSocketAddress("127.0.0.1", leport5 + 1000)));
 
-    protected void tearDown() throws Exception {
-        for(int i = 0; i < threads.size(); i++) {
-            LEThread leThread = threads.get(i);
-            ((FastLeaderElection) leThread.peer.getElectionAlg()).shutdown();
-            // shutdown() has to be explicitly called for every thread to
-            // make sure that resources are freed properly and all fixed network ports
-            // are available for other test cases
-            leThread.peer.shutdown();
+        LOG.info("creating QuorumPeer 1 port " + port1);
+        QuorumHierarchical hq1 = new QuorumHierarchical(qp); 
+        s1 = new QuorumPeer(peers, s1dir, s1dir, port1, 3, 1, tickTime, initLimit, syncLimit, hq1);
+        assertEquals(port1, s1.getClientPort());
+        
+        LOG.info("creating QuorumPeer 2 port " + port2);
+        QuorumHierarchical hq2 = new QuorumHierarchical(qp); 
+        s2 = new QuorumPeer(peers, s2dir, s2dir, port2, 3, 2, tickTime, initLimit, syncLimit, hq2);
+        assertEquals(port2, s2.getClientPort());
+        
+        LOG.info("creating QuorumPeer 3 port " + port3);
+        QuorumHierarchical hq3 = new QuorumHierarchical(qp); 
+        s3 = new QuorumPeer(peers, s3dir, s3dir, port3, 3, 3, tickTime, initLimit, syncLimit, hq3);
+        assertEquals(port3, s3.getClientPort());
+        
+        LOG.info("creating QuorumPeer 4 port " + port4);
+        QuorumHierarchical hq4 = new QuorumHierarchical(qp); 
+        s4 = new QuorumPeer(peers, s4dir, s4dir, port4, 3, 4, tickTime, initLimit, syncLimit, hq4);
+        assertEquals(port4, s4.getClientPort());
+        
+        LOG.info("creating QuorumPeer 5 port " + port5);
+        QuorumHierarchical hq5 = new QuorumHierarchical(qp); 
+        s5 = new QuorumPeer(peers, s5dir, s5dir, port5, 3, 5, tickTime, initLimit, syncLimit, hq5);
+        assertEquals(port5, s5.getClientPort());
+        LOG.info("start QuorumPeer 1");
+        s1.start();
+        LOG.info("start QuorumPeer 2");
+        s2.start();
+        LOG.info("start QuorumPeer 3");
+        s3.start();
+        LOG.info("start QuorumPeer 4");
+        s4.start();
+        LOG.info("start QuorumPeer 5");
+        s5.start();
+        LOG.info("started QuorumPeer 5");
+
+        LOG.info ("Closing ports " + hostPort);
+        for (String hp : hostPort.split(",")) {
+            assertTrue("waiting for server up",
+                       ClientBase.waitForServerUp(hp,
+                                    CONNECTION_TIMEOUT));
+            LOG.info(hp + " is accepting client connections");
         }
-        LOG.info("FINISHED " + getName());
+
+        // interesting to see what's there...
+        JMXEnv.dump();
+        // make sure we have these 5 servers listed
+        Set<String> ensureNames = new LinkedHashSet<String>();
+        for (int i = 1; i <= 5; i++) {
+            ensureNames.add("InMemoryDataTree");
+        }
+        for (int i = 1; i <= 5; i++) {
+            ensureNames.add("name0=ReplicatedServer_id" + i
+                 + ",name1=replica." + i + ",name2=");
+        }
+        for (int i = 1; i <= 5; i++) {
+            for (int j = 1; j <= 5; j++) {
+                ensureNames.add("name0=ReplicatedServer_id" + i
+                     + ",name1=replica." + j);
+            }
+        }
+        for (int i = 1; i <= 5; i++) {
+            ensureNames.add("name0=ReplicatedServer_id" + i);
+        }
+        JMXEnv.ensureAll(ensureNames.toArray(new String[ensureNames.size()]));
     }
 
-    class LEThread extends Thread {
-        int i;
-        QuorumPeer peer;
-        //int peerRound = 1;
+    @After
+    @Override
+    protected void tearDown() throws Exception {
+        LOG.info("TearDown started");
+        ct.tearDownAll();
 
-        LEThread(QuorumPeer peer, int i) {
-            this.i = i;
-            this.peer = peer;
-            LOG.info("Constructor: " + getName());
+        LOG.info("Shutting down server 1");
+        shutdown(s1);
+        LOG.info("Shutting down server 2");
+        shutdown(s2);
+        LOG.info("Shutting down server 3");
+        shutdown(s3);
+        LOG.info("Shutting down server 4");
+        shutdown(s4);
+        LOG.info("Shutting down server 5");
+        shutdown(s5);
+        
+        for (String hp : hostPort.split(",")) {
+            assertTrue("waiting for server down",
+                       ClientBase.waitForServerDown(hp,
+                                           ClientBase.CONNECTION_TIMEOUT));
+            LOG.info(hp + " is no longer accepting client connections");
         }
 
-        public void run() {
-            try {
-                Vote v = null;
-                while(true){
-
-                    //while(true) {
-                    peer.setPeerState(ServerState.LOOKING);
-                    LOG.info("Going to call leader election.");
-                    v = peer.getElectionAlg().lookForLeader();
-                    if(v == null){
-                        LOG.info("Thread " + i + " got a null vote");
-                        return;
-                    }
-
-                    /*
-                     * A real zookeeper would take care of setting the current vote. Here
-                     * we do it manually.
-                     */
-                    peer.setCurrentVote(v);
-
-                    LOG.info("Finished election: " + i + ", " + v.id);
-                    votes[i] = v;
-
-                    if((peer.getPeerState() == ServerState.FOLLOWING) ||
-                            (peer.getPeerState() == ServerState.LEADING)) break;
-                }
-                LOG.debug("Thread " + i + " votes " + v);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
+        JMXEnv.tearDown();
+
+        LOG.info("FINISHED " + getName());
+    }
+
+    protected void shutdown(QuorumPeer qp) {
+        try {
+            ((FastLeaderElection) qp.getElectionAlg()).shutdown();
+            LOG.info("Done with leader election");
+            qp.shutdown();
+            LOG.info("Done with quorum peer");
+            qp.join(30000);
+            if (qp.isAlive()) {
+                fail("QP failed to shutdown in 30 seconds");
             }
+        } catch (InterruptedException e) {
+            LOG.debug("QP interrupted", e);
         }
     }
 
-    @Test
-    public void testHierarchicalQuorum() throws Exception {
-        runTest(0);
+    protected ZooKeeper createClient()
+        throws IOException, InterruptedException
+    {
+        return createClient(hostPort);
     }
-    
-    @Test
-    public void testHierarchicalQuorumPartial() throws Exception {
-        runTest(3);
-    }
-    
-    private void runTest(int delta) throws Exception {
-        FastLeaderElection le[] = new FastLeaderElection[count];
-
-        LOG.info("TestHierarchicalQuorum: " + getName()+ ", " + count);
-        for(int i = 0; i < count; i++) {
-            int clientport = PortAssignment.unique();
-            peers.put(Long.valueOf(i),
-                    new QuorumServer(i,
-                            new InetSocketAddress(clientport),
-                    new InetSocketAddress(PortAssignment.unique())));
-            tmpdir[i] = ClientBase.createTmpDir();
-            port[i] = clientport;
-        }
 
-        for(int i = 0; i < (le.length - delta); i++) {
-            QuorumHierarchical hq = new QuorumHierarchical(qp);
-            QuorumPeer peer = new QuorumPeer(peers, tmpdir[i], tmpdir[i], port[i], 3, i, 2, 2, 2, hq);
-            peer.startLeaderElection();
-            LEThread thread = new LEThread(peer, i);
-            thread.start();
-            threads.add(thread);
-        }
-        LOG.info("Started threads " + getName());
+    protected ZooKeeper createClient(String hp)
+        throws IOException, InterruptedException
+    {
+        CountdownWatcher watcher = new CountdownWatcher();
+        return createClient(watcher, hp);
+    }
 
-        for(int i = 0; i < threads.size(); i++) {
-            threads.get(i).join(15000);
-            if (threads.get(i).isAlive()) {
-                fail("Threads didn't join");
-            }
-        }
+    @Test
+    public void testHierarchicalQuorum() throws Throwable {
+        ct.testHammerBasic();
     }
 }