ZOOKEEPER-764. Observer elected leader due to inconsistent voting view (henry via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/zookeeper/trunk@941473 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -46,6 +46,9 @@ BUGFIXES:
 
   ZOOKEEPER-737. some 4 letter words may fail with netcat (nc). (mahadev)
 
+  ZOOKEEPER-764. Observer elected leader due to inconsistent voting view 
+  (henry via mahadev)
+
 IMPROVEMENTS:
   ZOOKEEPER-724. Improve junit test integration - log harness information 
   (phunt via mahadev)

src/java/main/org/apache/zookeeper/server/quorum/FastLeaderElection.java

@@ -587,7 +587,7 @@ public class FastLeaderElection implements Election {
      * @return ServerState
      */
     private ServerState learningState(){
-        if(self.getPeerType() == LearnerType.PARTICIPANT){
+        if(self.getLearnerType() == LearnerType.PARTICIPANT){
             LOG.debug("I'm a participant: " + self.getId());
             return ServerState.FOLLOWING;
         }
@@ -603,7 +603,7 @@ public class FastLeaderElection implements Election {
      * @return long
      */
     private long getInitId(){
-        if(self.getPeerType() == LearnerType.PARTICIPANT)
+        if(self.getLearnerType() == LearnerType.PARTICIPANT)
             return self.getId();
         else return Long.MIN_VALUE;
     }
@@ -614,7 +614,7 @@ public class FastLeaderElection implements Election {
      * @return long
      */
     private long getInitLastLoggedZxid(){
-        if(self.getPeerType() == LearnerType.PARTICIPANT)
+        if(self.getLearnerType() == LearnerType.PARTICIPANT)
             return self.getLastLoggedZxid();
         else return Long.MIN_VALUE;
     }

src/java/main/org/apache/zookeeper/server/quorum/LeaderElection.java

@@ -226,7 +226,7 @@ public class LeaderElection implements Election  {
                             self.setCurrentVote(result.winner);
                             s.close();
                             Vote current = self.getCurrentVote();
-                            LOG.info("Found leader: my type is: " + self.getPeerType());
+                            LOG.info("Found leader: my type is: " + self.getLearnerType());
                             /*
                              * We want to make sure we implement the state machine
                              * correctly. If we are a PARTICIPANT, once a leader
@@ -234,7 +234,7 @@ public class LeaderElection implements Election  {
                              * FOLLOWING. However if we are an OBSERVER, it is an
                              * error to be elected as a Leader.
                              */
-                            if (self.getPeerType() == LearnerType.OBSERVER) {
+                            if (self.getLearnerType() == LearnerType.OBSERVER) {
                                 if (current.id == self.getId()) {
                                     // This should never happen!
                                     LOG.error("OBSERVER elected as leader!");

src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java

@@ -148,14 +148,24 @@ public class QuorumPeer extends Thread implements QuorumStats.Provider {
     /*
      * Default value of peer is participant
      */
-    private LearnerType peerType = LearnerType.PARTICIPANT;
+    private LearnerType learnerType = LearnerType.PARTICIPANT;
     
-    public LearnerType getPeerType() {
-        return peerType;
+    public LearnerType getLearnerType() {
+        return learnerType;
     }
     
-    public void setPeerType(LearnerType p) {
-        peerType = p;
+    /**
+     * Sets the LearnerType both in the QuorumPeer and in the peerMap
+     */
+    public void setLearnerType(LearnerType p) {
+        learnerType = p;
+        if (quorumPeers.containsKey(this.myid)) {
+            this.quorumPeers.get(myid).type = p;
+        } else {
+            LOG.error("Setting LearnerType to " + p + " but " + myid 
+                    + " not in QuorumPeers. ");
+        }
+        
     }
     /**
      * The servers that make up the cluster

src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerMain.java

@@ -138,7 +138,7 @@ public class QuorumPeerMain {
           quorumPeer.setQuorumVerifier(config.getQuorumVerifier());
           quorumPeer.setCnxnFactory(cnxnFactory);
           quorumPeer.setZKDatabase(new ZKDatabase(quorumPeer.getTxnFactory()));
-          quorumPeer.setPeerType(config.getPeerType());
+          quorumPeer.setLearnerType(config.getPeerType());
   
           quorumPeer.start();
           quorumPeer.join();

src/java/main/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java

@@ -57,6 +57,6 @@ public abstract class QuorumZooKeeperServer extends ZooKeeperServer {
         pwriter.print("quorumPort=");
         pwriter.println(self.quorumPeers.get(self.getId()).addr.getPort());
         pwriter.print("peerType=");
-        pwriter.println(self.getPeerType().ordinal());
+        pwriter.println(self.getLearnerType().ordinal());
     }
 }

src/java/test/org/apache/zookeeper/test/AsyncHammerTest.java

@@ -49,9 +49,8 @@ public class AsyncHammerTest extends ZKTestCase
 
     private volatile boolean bang;
 
-    @Before
-    public void setUp() throws Exception {
-        qb.setUp();
+    public void setUp(boolean withObservers) throws Exception {
+        qb.setUp(withObservers);
     }
 
     protected void restart() throws Exception {
@@ -63,7 +62,6 @@ public class AsyncHammerTest extends ZKTestCase
         qb.startServers();
     }
 
-    @After
     public void tearDown() throws Exception {
         LOG.info("Test clients shutting down");
         qb.tearDown();
@@ -171,6 +169,7 @@ public class AsyncHammerTest extends ZKTestCase
 
     @Test
     public void testHammer() throws Exception {
+        setUp(false);
         bang = true;
         LOG.info("Starting hammers");
         HammerThread[] hammers = new HammerThread[100];
@@ -197,12 +196,12 @@ public class AsyncHammerTest extends ZKTestCase
         // after restart
         LOG.info("Verifying hammers 2");
         qb.verifyRootOfAllServersMatch(qb.hostPort);
+        tearDown();
     }
 
     @Test
     public void testObserversHammer() throws Exception {
-        qb.tearDown();
-        qb.setUp(true);
+        setUp(true);
         bang = true;
         Thread[] hammers = new Thread[100];
         for (int i = 0; i < hammers.length; i++) {
@@ -217,6 +216,7 @@ public class AsyncHammerTest extends ZKTestCase
         }
         // before restart
         qb.verifyRootOfAllServersMatch(qb.hostPort);
+        tearDown();
     }
 
     @SuppressWarnings("unchecked")

src/java/test/org/apache/zookeeper/test/HierarchicalQuorumTest.java

@@ -166,7 +166,7 @@ public class HierarchicalQuorumTest extends ClientBase {
         QuorumHierarchical hq4 = new QuorumHierarchical(qp); 
         s4 = new QuorumPeer(peers, s4dir, s4dir, port4, 3, 4, tickTime, initLimit, syncLimit, hq4);
         if (withObservers) {
-            s4.setPeerType(QuorumPeer.LearnerType.OBSERVER);
+            s4.setLearnerType(QuorumPeer.LearnerType.OBSERVER);
         }
         Assert.assertEquals(port4, s4.getClientPort());
                        
@@ -174,7 +174,7 @@ public class HierarchicalQuorumTest extends ClientBase {
         QuorumHierarchical hq5 = new QuorumHierarchical(qp); 
         s5 = new QuorumPeer(peers, s5dir, s5dir, port5, 3, 5, tickTime, initLimit, syncLimit, hq5);
         if (withObservers) {
-            s5.setPeerType(QuorumPeer.LearnerType.OBSERVER);
+            s5.setLearnerType(QuorumPeer.LearnerType.OBSERVER);
         }
         Assert.assertEquals(port5, s5.getClientPort());
         

src/java/test/org/apache/zookeeper/test/LENonTerminateTest.java

@@ -169,7 +169,7 @@ public class LENonTerminateTest extends ZKTestCase {
                             self.setCurrentVote(result.winner);
                             s.close();
                             Vote current = self.getCurrentVote();
-                            LOG.info("Found leader: my type is: " + self.getPeerType());
+                            LOG.info("Found leader: my type is: " + self.getLearnerType());
                             /*
                              * We want to make sure we implement the state machine
                              * correctly. If we are a PARTICIPANT, once a leader
@@ -177,7 +177,7 @@ public class LENonTerminateTest extends ZKTestCase {
                              * FOLLOWING. However if we are an OBSERVER, it is an
                              * error to be elected as a Leader.
                              */
-                            if (self.getPeerType() == LearnerType.OBSERVER) {
+                            if (self.getLearnerType() == LearnerType.OBSERVER) {
                                 if (current.id == self.getId()) {
                                     // This should never happen!
                                     LOG.error("OBSERVER elected as leader!");

src/java/test/org/apache/zookeeper/test/QuorumBase.java

@@ -159,10 +159,16 @@ public class QuorumBase extends ClientBase {
         Assert.assertEquals(port5, s5.getClientPort());
         
         if (withObservers) {
-            s4.setPeerType(LearnerType.OBSERVER);
-            s5.setPeerType(LearnerType.OBSERVER);
+            s4.setLearnerType(LearnerType.OBSERVER);
+            s5.setLearnerType(LearnerType.OBSERVER);
         }
         
+        LOG.info("QuorumPeer 1 voting view: " + s1.getVotingView());
+        LOG.info("QuorumPeer 2 voting view: " + s2.getVotingView());
+        LOG.info("QuorumPeer 3 voting view: " + s3.getVotingView());
+        LOG.info("QuorumPeer 4 voting view: " + s4.getVotingView());
+        LOG.info("QuorumPeer 5 voting view: " + s5.getVotingView());       
+        
         LOG.info("start QuorumPeer 1");
         s1.start();
         LOG.info("start QuorumPeer 2");