ZOOKEEPER-1268. problems with read only mode, intermittent test failures and ERRORs in the log (phunt via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/zookeeper/trunk@1195854 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -436,6 +436,9 @@ BUGFIXES:
   ZOOKEEPER-1181. Fix problems with Kerberos TGT renewal.
   (Eugene Koontz via mahadev)
 
+  ZOOKEEPER-1268. problems with read only mode, intermittent test failures 
+  and ERRORs in the log (phunt via mahadev)
+
 IMPROVEMENTS:
   ZOOKEEPER-724. Improve junit test integration - log harness information 
   (phunt via mahadev)

src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml

@@ -1015,6 +1015,35 @@ server.3=zoo3:2888:3888</programlisting>
         </variablelist>
       </section>
 
+      <section>
+        <title>Experimental Options/Features</title>
+
+        <para>New features that are currently considered experimental.</para>
+
+        <variablelist>
+          <varlistentry>
+            <term>Read Only Mode Server</term>
+
+            <listitem>
+              <para>(Java system property: <emphasis
+              role="bold">readonlymode.enabled</emphasis>)</para>
+
+              <para><emphasis role="bold">New in 3.4.0:</emphasis>
+              Setting this value to true enables Read Only Mode server
+              support (disabled by default). ROM allows clients
+              sessions which requested ROM support to connect to the
+              server even when the server might be partitioned from
+              the quorum. In this mode ROM clients can still read
+              values from the ZK service, but will be unable to write
+              values and see changes from other clients. See
+              ZOOKEEPER-784 for more details.
+              </para>
+            </listitem>
+          </varlistentry>
+
+        </variablelist>
+      </section>
+
       <section>
         <title>Unsafe Options</title>
 

src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java

@@ -664,39 +664,53 @@ public class QuorumPeer extends Thread implements QuorumStats.Provider {
                 case LOOKING:
                     LOG.info("LOOKING");
 
-                    // Create read-only server but don't start it immediately
-                    final ReadOnlyZooKeeperServer roZk = new ReadOnlyZooKeeperServer(logFactory, this, this.zkDb);
-
-                    // Instead of starting roZk immediately, wait some grace
-                    // period before we decide we're partitioned.
-                    //
-                    // Thread is used here because otherwise it would require
-                    // changes in each of election strategy classes which is
-                    // unnecessary code coupling.
-                    Thread roZkMgr = new Thread() {
-                        public void run() {
-                            try {
-                                // lower-bound grace period to 2 secs
-                                sleep(Math.max(2000, tickTime));
-                                if (ServerState.LOOKING.equals(getPeerState())) {
-                                    roZk.startup();
+                    if (Boolean.getBoolean("readonlymode.enabled")) {
+                        LOG.info("Attempting to start ReadOnlyZooKeeperServer");
+
+                        // Create read-only server but don't start it immediately
+                        final ReadOnlyZooKeeperServer roZk =
+                            new ReadOnlyZooKeeperServer(logFactory, this, this.zkDb);
+    
+                        // Instead of starting roZk immediately, wait some grace
+                        // period before we decide we're partitioned.
+                        //
+                        // Thread is used here because otherwise it would require
+                        // changes in each of election strategy classes which is
+                        // unnecessary code coupling.
+                        Thread roZkMgr = new Thread() {
+                            public void run() {
+                                try {
+                                    // lower-bound grace period to 2 secs
+                                    sleep(Math.max(2000, tickTime));
+                                    if (ServerState.LOOKING.equals(getPeerState())) {
+                                        roZk.startup();
+                                    }
+                                } catch (InterruptedException e) {
+                                    LOG.info("Interrupted while attempting to start ReadOnlyZooKeeperServer, not started");
+                                } catch (Exception e) {
+                                    LOG.error("FAILED to start ReadOnlyZooKeeperServer", e);
                                 }
-                            } catch (Exception e) {
-                                LOG.error("FAILED to start ReadOnlyZooKeeperServer", e);
                             }
+                        };
+                        try {
+                            roZkMgr.start();
+                            setCurrentVote(makeLEStrategy().lookForLeader());
+                        } catch (Exception e) {
+                            LOG.warn("Unexpected exception", e);
+                            setPeerState(ServerState.LOOKING);
+                        } finally {
+                            // If the thread is in the the grace period, interrupt
+                            // to come out of waiting.
+                            roZkMgr.interrupt();
+                            roZk.shutdown();
                         }
-                    };
-                    try {
-                        roZkMgr.start();
-                        setCurrentVote(makeLEStrategy().lookForLeader());
-                    } catch (Exception e) {
-                        LOG.warn("Unexpected exception",e);
-                        setPeerState(ServerState.LOOKING);
-                    } finally {
-                        // If the thread is in the the grace period, interrupt
-                        // to come out of waiting.
-                        roZkMgr.interrupt();
-                        roZk.shutdown();
+                    } else {
+                        try {
+                            setCurrentVote(makeLEStrategy().lookForLeader());
+                        } catch (Exception e) {
+                            LOG.warn("Unexpected exception", e);
+                            setPeerState(ServerState.LOOKING);
+                        }                        
                     }
                     break;
                 case OBSERVING:

src/java/test/org/apache/zookeeper/test/ReadOnlyModeTest.java

@@ -52,11 +52,13 @@ public class ReadOnlyModeTest extends ZKTestCase {
 
     @Before
     public void setUp() throws Exception {
+        System.setProperty("readonlymode.enabled", "true");
         qu.startQuorum();
     }
 
     @After
     public void tearDown() throws Exception {
+        System.setProperty("readonlymode.enabled", "false");
         qu.tearDown();
     }
 
@@ -133,9 +135,10 @@ public class ReadOnlyModeTest extends ZKTestCase {
         long start = System.currentTimeMillis();
         while (!(zk.getState() == States.CONNECTEDREADONLY)) {
             Thread.sleep(200);
+            // FIXME this was originally 5 seconds, but realistically, on random/slow/virt hosts, there is no way to guarantee this
             Assert.assertTrue("Can't connect to the server", System
                     .currentTimeMillis()
-                    - start < 5000);
+                    - start < 30000);
         }
 
         // At this point states list should contain, in the given order,