ZOOKEEPER-3179: Add snapshot compression to reduce the disk IO

Author: Yisong Yue <yisongyue@fb.com>

Reviewers: andor@apache.org

Closes #690 from yisong-yue/ZOOKEEPER-3179

build.xml

@@ -17,7 +17,7 @@
    limitations under the License.
 -->
 
-<project name="ZooKeeper" default="jar" 
+<project name="ZooKeeper" default="jar"
 xmlns:ivy="antlib:org.apache.ivy.ant"
 xmlns:artifact="antlib:org.apache.maven.artifact.ant"
 xmlns:maven="antlib:org.apache.maven.artifact.ant"
@@ -62,6 +62,8 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     <property name="kerby.version" value="1.1.0"/>
 
     <property name="clover.version" value="4.2.1" />
+
+    <property name="snappy.version" value="1.1.7"/>
     <property name="json.version" value="1.1.1"/>
 
     <!-- ====================================================== -->
@@ -74,7 +76,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     <property name="name" value="zookeeper" />
 
     <property environment="env"/>
-    
+
     <property name="version-major" value="3" />
     <property name="version-minor" value="6" />
     <property name="version-patch" value="0" />
@@ -86,7 +88,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     <property name="revision.dir" value="${basedir}/.revision" />
     <property name="revision.properties" value="revision.properties" />
     <property file="${basedir}/.revision/${revision.properties}" />
-    
+
     <property name="javac.target" value="1.8" />
     <property name="javac.source" value="1.8" />
     <property name="build.encoding" value="utf8" />
@@ -207,7 +209,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     <property name="jdiff.build.dir" value="${build.docs}/jdiff"/>
     <property name="jdiff.xml.dir" value="${lib.dir}/jdiff"/>
     <property name="jdiff.stable" value="3.1.1"/>
-    <property name="jdiff.stable.javadoc" 
+    <property name="jdiff.stable.javadoc"
             value="http://hadoop.apache.org/zookeeper/docs/r${jdiff.stable}/api/"/>
 
     <!-- eclipse property set -->
@@ -223,14 +225,14 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     <property name="staging_repo_id" value="apache.staging.https"/>
     <property name="wagon-http.version" value="2.4"/>
     <property name="snapshots_repo_id" value="apache.snapshots.https"/>
-    <property name="asfrepo" value="https://repository.apache.org"/> 
-    <property name="snapshots_repo_url" 
-      value="${asfrepo}/content/repositories/snapshots"/> 
+    <property name="asfrepo" value="https://repository.apache.org"/>
+    <property name="snapshots_repo_url"
+      value="${asfrepo}/content/repositories/snapshots"/>
     <property name="staging_repo_url"
-      value="${asfrepo}/service/local/staging/deploy/maven2"/> 
-    <property name="gpg-plugin" 
+      value="${asfrepo}/service/local/staging/deploy/maven2"/>
+    <property name="gpg-plugin"
       value="org.apache.maven.plugins:maven-gpg-plugin:1.4:sign-and-deploy-file"/>
-    <property name="deploy-plugin" 
+    <property name="deploy-plugin"
       value="org.apache.maven.plugins:maven-deploy-plugin:2.8.1:deploy-file"/>
     <property name="main-jar" value="${dist.maven.dir}/${final.name}.jar"/>
     <property name="tests-jar" value="${dist.maven.dir}/${final.name}-tests.jar"/>
@@ -294,7 +296,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     <!-- ====================================================== -->
     <!-- Generate and compile the Java files                    -->
     <!-- ====================================================== -->
-    <target name="init">    
+    <target name="init">
         <mkdir dir="${build.classes}" />
 
         <mkdir dir="${ivy.lib}"/>
@@ -309,8 +311,8 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
             <format property="year" pattern="yyyy" timezone="GMT"/>
         </tstamp>
     </target>
-    
-	
+
+
     <target name="generate_jute_parser" depends="init,ivy-retrieve,ivy-retrieve-javacc">
       <property name="jute_javacc.dir" value="${build.dir}/jute_compiler" />
       <property name="jute_javacc.packagedir" value="/org/apache/jute/compiler/generated" />
@@ -324,7 +326,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
           javacchome="${ivy.javacc.lib}"
           />
     </target>
-     
+
     <target name="jute" depends="generate_jute_parser">
         <javac srcdir="${jute_javacc.dir}" destdir="${build.classes}" includeantruntime="false"
             target="${javac.target}" source="${javac.source}"
@@ -352,7 +354,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
                 <pathelement path="${jute_javacc.dir}" />
             </classpath>
         </java>
-        
+
         <java classname="org.apache.jute.compiler.generated.Rcc" fork="true" dir="${csrc_generated.dir}">
             <arg value="-l" />
             <arg value="c" />
@@ -372,13 +374,13 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
             <src path="${java.server.src.dir}" />
         </javac>
     </target>
-    
+
     <target name="git-revision" unless="lastRevision">
         <mkdir dir="${revision.dir}" />
         <condition property="shell.name" value="cmd" else="sh">
       	    <os family="windows"/>
         </condition>
-        <condition property="revision.cmd.line" 
+        <condition property="revision.cmd.line"
         	value="/c ${java.server.resources.dir}\lastRevision.bat" else="${java.server.resources.dir}/lastRevision.sh">
       	    <os family="windows"/>
         </condition>
@@ -387,10 +389,10 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
         </exec>
         <property file="${revision.dir}/${revision.properties}" />
     </target>
-    
+
     <target name="version-info" depends="ver-gen,git-revision">
         <mkdir dir="${src_generated.dir}" />
-        <java classname="org.apache.zookeeper.version.util.VerGen" fork="true" 
+        <java classname="org.apache.zookeeper.version.util.VerGen" fork="true"
                 dir="${src_generated.dir}">
             <arg value="${version}" />
             <arg value="${lastRevision}" />
@@ -400,12 +402,12 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
             </classpath>
         </java>
     </target>
-    
+
     <target name="build-generated" depends="compile_jute,version-info,process-template,ivy-retrieve" >
         <javac srcdir="${src_generated.dir}" destdir="${build.classes}" includeantruntime="false"
             target="${javac.target}" source="${javac.source}" debug="on" encoding="${build.encoding}" classpath="${ivy.lib}/audience-annotations-${audience-annotations.version}.jar" />
     </target>
-    
+
     <target name="ivy-download" unless="ivy.jar.exists" depends="init">
       <delete dir="${lib.dir}"
               includes="ivy-*.jar" excludes="ivy-${ivy.version}.jar"/>
@@ -417,7 +419,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
       <taskdef resource="org/apache/ivy/ant/antlib.xml"
                uri="antlib:org.apache.ivy.ant" classpathref="java.classpath"/>
       <!-- ensure that ivy taskdef is only run once, otw ant will error -->
-      <property name="ivy.initialized" value="true"/> 
+      <property name="ivy.initialized" value="true"/>
     </target>
 
     <target name="ivy-init" depends="ivy-download,ivy-taskdef">
@@ -491,7 +493,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
         </filterchain>
       </copy>
     </target>
-    
+
     <target name="ivy-retrieve-mvn-ant-task" depends="init,ivy-init">
       <ivy:retrieve settingsRef="${ant.project.name}" conf="mvn-ant-task"
                     pattern="${ivy.lib}/[artifact]-[revision].[ext]"/>
@@ -580,7 +582,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
         <link href="${javadoc.link.java}"/>
         <classpath refid="java.classpath"/>
       </javadoc>
-    </target>	
+    </target>
 
     <target name="javadoc" depends="jar" description="Generate javadoc">
       <mkdir dir="${build.javadoc}"/>
@@ -624,17 +626,17 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
         <classpath>
 		<fileset dir="${basedir}">
 			<include name="${jar.name}"/>
-		</fileset> 
+		</fileset>
         <path refid="java.classpath"/>
 	</classpath>
       </javadoc>
-    </target>	
+    </target>
 
     <!-- ====================================================== -->
     <!-- Make zookeeper.jar                                     -->
     <!-- ====================================================== -->
-	
-	
+
+
     <target name="jar" depends="compile">
         <java classname="org.apache.zookeeper.Version" fork="true"
               outputproperty="revision" errorproperty="revision.error" failonerror="true">
@@ -645,16 +647,16 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
             </classpath>
         </java>
         <exec executable="hostname" outputproperty="host.name"/>
-	    
+
     	<!-- Ensure that OSGi versions order properly whatever suffix is used by -->
     	<!-- using DEV and REL at the start of the qualifier. DEV versions also -->
     	<!-- use a timestamp to ensure later snapshots have higher versions -->
-	    <condition property="version-osgi" 
+	    <condition property="version-osgi"
 		       value="${version-base}.DEV-${DSTAMP}-${TSTAMP}-${version-suffix}"
 		       else="${version-base}.REL-${version-suffix}">
 	        <contains string="${version-suffix}" substring="SNAPSHOT" />
 	    </condition>
-        
+
     	<jar jarfile="${build.dir}/${final.name}.jar">
             <fileset file="LICENSE.txt" />
             <fileset dir="${build.classes}" excludes="**/.generated"/>
@@ -667,11 +669,11 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
                 <attribute name="Built-At" value="${build.time}"/>
                 <attribute name="Built-On" value="${host.name}" />
                 <attribute name="Implementation-Title" value="org.apache.zookeeper"/>
-                <attribute name="Implementation-Version" value="${revision}"/> 
+                <attribute name="Implementation-Version" value="${revision}"/>
                 <attribute name="Implementation-Vendor" value="The Apache Software Foundation"/>
-            	
+
             	<!-- The following are OSGi manifest headers -->
-            	<!-- currently hardcoded, when things get more complicated we could use BND 
+            	<!-- currently hardcoded, when things get more complicated we could use BND
             	http://www.aqute.biz/Code/Bnd to generate them -->
                 <attribute name="Bundle-Vendor" value="The Apache Software Foundation"/>
                 <attribute name="Bundle-Name" value="ZooKeeper Bundle"/>
@@ -685,7 +687,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
             </manifest>
         </jar>
     </target>
-    
+
     <!-- ====================================================== -->
     <!-- Make zookeeper-bin.jar                                 -->
     <!-- ====================================================== -->
@@ -708,11 +710,11 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
                 <attribute name="Built-At" value="${build.time}"/>
                 <attribute name="Built-On" value="${host.name}" />
                 <attribute name="Implementation-Title" value="org.apache.zookeeper"/>
-                <attribute name="Implementation-Version" value="${revision}"/> 
+                <attribute name="Implementation-Version" value="${revision}"/>
                 <attribute name="Implementation-Vendor" value="The Apache Software Foundation"/>
-            	
+
             	<!-- The following are OSGi manifest headers -->
-            	<!-- currently hardcoded, when things get more complicated we could use BND 
+            	<!-- currently hardcoded, when things get more complicated we could use BND
             	http://www.aqute.biz/Code/Bnd to generate them -->
                 <attribute name="Bundle-Vendor" value="The Apache Software Foundation"/>
                 <attribute name="Bundle-Name" value="ZooKeeper Bundle"/>
@@ -741,7 +743,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
                 <attribute name="Built-At" value="${build.time}"/>
                 <attribute name="Built-On" value="${host.name}" />
                 <attribute name="Implementation-Title" value="org.apache.zookeeper"/>
-                <attribute name="Implementation-Version" value="${revision}"/> 
+                <attribute name="Implementation-Version" value="${revision}"/>
                 <attribute name="Implementation-Vendor" value="The Apache Software Foundation"/>
             </manifest>
         </jar>
@@ -759,7 +761,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
                 <attribute name="Built-At" value="${build.time}"/>
                 <attribute name="Built-On" value="${host.name}" />
                 <attribute name="Implementation-Title" value="org.apache.zookeeper"/>
-                <attribute name="Implementation-Version" value="${revision}"/> 
+                <attribute name="Implementation-Version" value="${revision}"/>
                 <attribute name="Implementation-Vendor" value="The Apache Software Foundation"/>
             </manifest>
         </jar>
@@ -777,7 +779,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
                 <attribute name="Built-At" value="${build.time}"/>
                 <attribute name="Built-On" value="${host.name}" />
                 <attribute name="Implementation-Title" value="org.apache.zookeeper"/>
-                <attribute name="Implementation-Version" value="${revision}"/> 
+                <attribute name="Implementation-Version" value="${revision}"/>
                 <attribute name="Implementation-Vendor" value="The Apache Software Foundation"/>
             </manifest>
         </jar>
@@ -788,7 +790,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     <!-- ================================================================== -->
     <!--                                                                    -->
     <!-- ================================================================== -->
-    <target name="package" 
+    <target name="package"
             depends="jar,bin-jar,src-jar,javadoc-jar,test-jar,api-report,create-cppunit-configure,compile-test"
             description="Build distribution">
       <mkdir dir="${dist.dir}"/>
@@ -813,9 +815,9 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
         <property name="package.share" value=""/>
         <fileset file="${contrib.dir}/build.xml"/>
         <fileset file="${recipes.dir}/build.xml"/>
-      </subant>  	
+      </subant>
 
-      <copy todir="${dist.dir}"> 
+      <copy todir="${dist.dir}">
         <fileset file="${build.dir}/${final.name}.jar"/>
       </copy>
 
@@ -825,8 +827,8 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
       <mkdir dir="${dist.maven.dir}"/>
 
       <copy file="${build.dir}/${final.name}-bin.jar"
-            tofile="${dist.maven.dir}/${final.name}.jar"/> 
-      <copy todir="${dist.maven.dir}"> 
+            tofile="${dist.maven.dir}/${final.name}.jar"/>
+      <copy todir="${dist.maven.dir}">
         <fileset file="${build.dir}/${final.name}-sources.jar"/>
         <fileset file="${build.dir}/${final.name}-javadoc.jar"/>
       </copy>
@@ -848,7 +850,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
       <checksum file="${dist.maven.dir}/${final.name}.pom" algorithm="sha1"/>
 
       <copy file="${build.dir}/${final.name}-test.jar"
-            tofile="${dist.maven.dir}/${final.name}-tests.jar"/> 
+            tofile="${dist.maven.dir}/${final.name}-tests.jar"/>
       <checksum file="${dist.maven.dir}/${final.name}-tests.jar" algorithm="sha1"/>
       <checksum file="${dist.maven.dir}/${final.name}-tests.jar" algorithm="md5"/>
 
@@ -895,7 +897,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
       <copy todir="${dist.dir}/zookeeper-server/src/main/resources">
         <fileset file="${java.server.resources.dir}/pom.template"/>
       </copy>
-  	  
+
       <chmod perm="ugo+x" type="file" parallel="false">
         <fileset dir="${dist.dir}/bin"/>
         <fileset dir="${dist.dir}/zookeeper-contrib/">
@@ -934,7 +936,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
         <fileset file="${recipes.dir}/build.xml"/>
       </subant>
 
-      <copy todir="${dist.dir}/share/zookeeper"> 
+      <copy todir="${dist.dir}/share/zookeeper">
         <fileset file="${build.dir}/${final.name}.jar"/>
       </copy>
 
@@ -944,12 +946,12 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
       <mkdir dir="${dist.maven.dir}"/>
 
       <copy file="${build.dir}/${final.name}-bin.jar"
-            tofile="${dist.maven.dir}/${final.name}.jar"/> 
-      <copy todir="${dist.maven.dir}"> 
+            tofile="${dist.maven.dir}/${final.name}.jar"/>
+      <copy todir="${dist.maven.dir}">
         <fileset file="${build.dir}/${final.name}-sources.jar"/>
         <fileset file="${build.dir}/${final.name}-javadoc.jar"/>
       </copy>
-      
+
       <checksum file="${dist.maven.dir}/${final.name}.jar" algorithm="md5"/>
       <checksum file="${dist.maven.dir}/${final.name}.jar" algorithm="sha1"/>
       <checksum file="${dist.maven.dir}/${final.name}-sources.jar" algorithm="md5"/>
@@ -965,8 +967,8 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
       </ivy:makepom>
       <checksum file="${dist.maven.dir}/${name}.pom" algorithm="md5"/>
       <checksum file="${dist.maven.dir}/${name}.pom" algorithm="sha1"/>
-      
-      
+
+
       <copy file="${build.dir}/${final.name}-test.jar"
             tofile="${dist.maven.dir}/${final.name}-tests.jar"/>
       <checksum file="${dist.maven.dir}/${final.name}-tests.jar" algorithm="sha1"/>
@@ -1153,7 +1155,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
 
     <target name="signanddeploy" if="staging">
       <deploy repo-url="${staging_repo_url}" repo-id="${staging_repo_id}" plugin="${gpg-plugin}" profile="-Pgpg"/>
-    </target> 
+    </target>
 
     <target name="simpledeploy" unless="staging">
       <deploy plugin="${deploy-plugin}"/>
@@ -1162,7 +1164,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     <!-- ====================================================== -->
     <!-- mvn-install. Installing the jar and pom file to .m2    -->
     <!-- ====================================================== -->
-     
+
      <target name="mvn-taskdef" depends="ivy-retrieve-mvn-ant-task">
         <typedef resource="org/apache/maven/artifact/ant/antlib.xml"
         uri="antlib:org.apache.maven.artifact.ant" classpathref="mvn-ant-task-classpath"/>
@@ -1174,11 +1176,11 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
        <echo message="${dist.maven.dir}/${final.name}-sources.jar" />
        <echo message="${dist.maven.dir}/${final.name}-javadoc.jar" />
        <echo message="${dist.maven.dir}/${final.name}-tests.jar" />
-  
+
        <artifact:pom id="zookeeper-pom" file="${dist.maven.dir}/${name}.pom"/>
        <echo>The version is ${zookeeper-pom.version}</echo>
        <echo message="${dist.maven.dir}/${final.name}.jar" />
-     
+
        <artifact:install file="${dist.maven.dir}/${final.name}.jar">
          <pom refid="zookeeper-pom" />
          <attach file="${dist.maven.dir}/${final.name}.jar" type="jar"/>
@@ -1199,7 +1201,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
       <delete dir="${csrc_generated.dir}" />
       <delete file="${lib.dir}/Null.java"/>
       <delete file="${lib.dir}/rats.jar" />
-      <delete file="${jdiff.xml.dir}/${name}_${version}.xml"/>	
+      <delete file="${jdiff.xml.dir}/${name}_${version}.xml"/>
       <delete file="${jar.name}" />
       <delete dir="${distribution}"/>
       <delete dir="${revision.dir}"/>
@@ -1209,9 +1211,9 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     </target>
 
     <target name="clean-contrib">
-      <subant target="clean">        
+      <subant target="clean">
         <fileset file="${contrib.dir}/build.xml"/>
-      </subant>  	
+      </subant>
     </target>
 
    <target name="clean-recipes">
@@ -1376,7 +1378,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
       <condition property="need.cppunit.configure">
         <not> <available file="${c.src.dir}/configure"/> </not>
       </condition>
-    </target>	
+    </target>
 
     <target name="verify-cppunit-makefile-gcov">
       <fileset id="fileset.makefile.gcov.enabled" dir="${test.cppunit.dir}" erroronmissingdir="false">
@@ -1395,11 +1397,11 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
       </condition>
       <echo message="delete.cppunit.makefile = ${delete.cppunit.makefile}"/>
     </target>
-    
+
     <target name="delete-cppunit-makefile" if="delete.cppunit.makefile">
       <delete file="${test.cppunit.dir}/Makefile"/>
     </target>
-    
+
     <target name="check-cppunit-makefile" depends="init,verify-cppunit-makefile-gcov,delete-cppunit-makefile" >
     	<condition property="need.cppunit.makefile">
        		<not> <available file="${test.cppunit.dir}/Makefile"/> </not>
@@ -1409,7 +1411,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     <!--
        1. If we have a Makefile it will handle up-to-date check and also
           regenerate the configure script if missing. (done)
-       2. If we don't have a Makefile use the configure script to 
+       2. If we don't have a Makefile use the configure script to
           regenerate it. (done)
        3. If we don't have a Makefile nor a configure script then it's
           last resort and run autoreconf, then configure (done)
@@ -1424,7 +1426,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
       </exec>
     </target>
 
-    <target name="create-cppunit-makefile" depends="check-cppunit-makefile" 
+    <target name="create-cppunit-makefile" depends="check-cppunit-makefile"
                                            if="need.cppunit.makefile">
       <antcall target="create-cppunit-configure">
         <param name="cppunit" value="true"/>
@@ -1447,7 +1449,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
 	<antcall target="test-cppunit">
 		<param name="cppunit" value="true"/>
 	</antcall>
-    </target>			
+    </target>
 
 
     <target name="test-cppunit"
@@ -1463,7 +1465,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
 		<arg line="clean check"/>
 	</exec>
     </target>
-   
+
     <target name="test-unit-category">
       <property name="test.category" value="Unit"/>
     </target>
@@ -1487,11 +1489,11 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
 	<antcall target="test-core"/>
 	<antcall target="test-contrib"/>
     </target>
- 
+
     <target name="test-contrib" description="to run contrib tests">
 	<!-- yet to implement -->
     </target>
-    
+
     <target name="test-core-java" depends="test-init, test-category, junit.run"/>
 
     <target name="test-core-cppunit" depends="test-init, test-category, call-test-cppunit"/>
@@ -1587,10 +1589,10 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
         <property name="findbugs.exclude.file" value="${server.test.resource.dir}/findbugsExcludeFile.xml" />
         <property name="findbugs.report.htmlfile" value="${findbugs.out.dir}/zookeeper-findbugs-report.html" />
         <property name="findbugs.report.xmlfile" value="${findbugs.out.dir}/zookeeper-findbugs-report.xml" />
-        <taskdef name="findbugs" classname="edu.umd.cs.findbugs.anttask.FindBugsTask" 
+        <taskdef name="findbugs" classname="edu.umd.cs.findbugs.anttask.FindBugsTask"
                 classpath="${findbugs.home}/lib/findbugs-ant.jar" />
         <mkdir dir="${findbugs.out.dir}" />
-        <findbugs home="${findbugs.home}" output="xml:withMessages" excludeFilter="${findbugs.exclude.file}" 
+        <findbugs home="${findbugs.home}" output="xml:withMessages" excludeFilter="${findbugs.exclude.file}"
                 outputFile="${findbugs.report.xmlfile}" effort="max" jvmargs="-Xmx512M">
             <auxClasspath>
                 <fileset dir="${ivy.lib}">
@@ -1600,7 +1602,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
             <sourcePath path="${java.server.src.dir}" />
             <class location="${build.dir}/${final.name}.jar" />
         </findbugs>
-        <xslt style="${findbugs.home}/src/xsl/default.xsl" in="${findbugs.report.xmlfile}" 
+        <xslt style="${findbugs.home}/src/xsl/default.xsl" in="${findbugs.report.xmlfile}"
                 out="${findbugs.report.htmlfile}" />
     </target>
 
@@ -1633,7 +1635,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     </target>
 
     <target name="cobertura-test" depends="test-init,cobertura-instrument">
-        <junit showoutput="${test.output}" printsummary="yes" haltonfailure="no" fork="yes" 
+        <junit showoutput="${test.output}" printsummary="yes" haltonfailure="no" fork="yes"
                maxmemory="${test.junit.maxmem}" dir="${basedir}" timeout="${test.timeout}"
                errorProperty="tests.failed" failureProperty="tests.failed">
             <sysproperty key="build.test.dir" value="${test.tmp.dir}" />
@@ -1805,12 +1807,12 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     </target>
 
     <target name="findbugs.check" depends="check-for-findbugs" unless="findbugs.present">
-    	<fail message="'findbugs.home' is not defined. Please pass -Dfindbugs.home=&lt;base of Findbugs installation&gt; 
+    	<fail message="'findbugs.home' is not defined. Please pass -Dfindbugs.home=&lt;base of Findbugs installation&gt;
 		to Ant on the command-line." />
     </target>
 
     <target name="patch.check" unless="patch.file">
-  	<fail message="'patch.file' is not defined. Please pass -Dpatch.file=&lt;location of patch file&gt; 
+  	<fail message="'patch.file' is not defined. Please pass -Dpatch.file=&lt;location of patch file&gt;
 		to Ant on the command-line." />
     </target>
 
@@ -1894,17 +1896,17 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
          </classpath>
        </javadoc>
      </target>
-     
+
      <target name="write-null">
        <exec executable="touch">
          <arg value="${jdiff.home}/Null.java"/>
        </exec>
-     </target> 
+     </target>
 
      <target name="api-report" depends="api-xml">
        <mkdir dir="${jdiff.build.dir}"/>
        <javadoc destdir="${jdiff.build.dir}"
-                excludepackagenames="org.apache.jute"	
+                excludepackagenames="org.apache.jute"
                 sourceFiles="${jdiff.home}/Null.java">
          <fileset dir="${java.server.src.dir}"/>
          <doclet name="jdiff.JDiff"
@@ -1979,7 +1981,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
                    output="${build.dir.eclipse-test-classes}" />
            <source path="${bench.src.dir}"
                    output="${build.dir.eclipse-test-classes}" />
-                   
+
            <output path="${build.dir.eclipse-main-classes}" />
            <library pathref="default.path.id" exported="true" />
            <library pathref="junit.path.id" exported="false" />

ivy.xml

@@ -76,7 +76,7 @@
         <exclude org="commons-cli" module="commons-cli"/>
     </dependency>
     <!-- force the tests to pull the latest commons-collections jar -->
-    <dependency org="commons-collections" name="commons-collections" 
+    <dependency org="commons-collections" name="commons-collections"
                 rev="${commons-collections.version}" conf="test->default"/>
 
     <dependency org="org.bouncycastle" name="bcprov-jdk15on" rev="${bouncycastle.version}" conf="test->default"/>
@@ -87,7 +87,7 @@
     <dependency org="xerces" name="xerces" rev="${xerces.version}"
                 conf="jdiff->default"/>
 
-    <dependency org="org.apache.rat" name="apache-rat-tasks" 
+    <dependency org="org.apache.rat" name="apache-rat-tasks"
                 rev="${apache-rat-tasks.version}" conf="releaseaudit->default">
         <exclude org="commons-collections" module="commons-collections"/>
     </dependency>
@@ -141,6 +141,7 @@
                   conf="optional->default"/>
     <dependency org="com.fasterxml.jackson.core" name="jackson-databind"
                 rev="${jackson.version}" conf="optional->default"/>
+    <dependency org="org.xerial.snappy" name="snappy-java" rev="${snappy.version}"/>
 
     <dependency org="org.openclover" name="clover" rev="${clover.version}" conf="clover->default"/>
 

zookeeper-server/pom.xml

@@ -142,6 +142,12 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.xerial.snappy</groupId>
+      <artifactId>snappy-java</artifactId>
+      <version>1.1.7</version>
+      <scope>compile</scope>
+    </dependency>
   </dependencies>
 
   <build>

zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotFormatter.java

@@ -19,8 +19,8 @@
 package org.apache.zookeeper.server;
 
 import java.io.BufferedInputStream;
-import java.io.FileInputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Base64;
@@ -37,6 +37,7 @@ import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.zookeeper.ZKUtil;
 import org.apache.zookeeper.data.StatPersisted;
 import org.apache.zookeeper.server.persistence.FileSnap;
+import org.apache.zookeeper.server.persistence.SnapStream;
 import org.apache.zookeeper.server.persistence.Util;
 import org.json.simple.JSONValue;
 
@@ -97,9 +98,7 @@ public class SnapshotFormatter {
     public void run(String snapshotFileName, boolean dumpData, boolean dumpJson)
         throws IOException {
         File snapshotFile = new File(snapshotFileName);
-        try (InputStream is = new CheckedInputStream(
-                new BufferedInputStream(new FileInputStream(snapshotFileName)),
-                new Adler32())) {
+        try (InputStream is = SnapStream.getInputStream(snapshotFile)) {
             InputArchive ia = BinaryInputArchive.getArchive(is);
 
             FileSnap fileSnap = new FileSnap(null);

zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileSnap.java

@@ -17,11 +17,7 @@
  */
 
 package org.apache.zookeeper.server.persistence;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -29,7 +25,6 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.zip.Adler32;
 import java.util.zip.CheckedInputStream;
 import java.util.zip.CheckedOutputStream;
 
@@ -82,15 +77,10 @@ public class FileSnap implements SnapShot {
         for (int i = 0, snapListSize = snapList.size(); i < snapListSize; i++) {
             snap = snapList.get(i);
             LOG.info("Reading snapshot " + snap);
-            try (InputStream snapIS = new BufferedInputStream(new FileInputStream(snap));
-                 CheckedInputStream crcIn = new CheckedInputStream(snapIS, new Adler32())) {
-                InputArchive ia = BinaryInputArchive.getArchive(crcIn);
+            try (CheckedInputStream snapIS = SnapStream.getInputStream(snap)) {
+                InputArchive ia = BinaryInputArchive.getArchive(snapIS);
                 deserialize(dt, sessions, ia);
-                long checkSum = crcIn.getChecksum().getValue();
-                long val = ia.readLong("val");
-                if (val != checkSum) {
-                    throw new IOException("CRC corruption in snapshot :  " + snap);
-                }
+                SnapStream.checkSealIntegrity(snapIS, ia);
                 foundValid = true;
                 break;
             } catch (IOException e) {
@@ -136,12 +126,12 @@ public class FileSnap implements SnapShot {
     }
 
     /**
-     * find the last (maybe) valid n snapshots. this does some 
+     * find the last (maybe) valid n snapshots. this does some
      * minor checks on the validity of the snapshots. It just
      * checks for / at the end of the snapshot. This does
      * not mean that the snapshot is truly valid but is
-     * valid with a high probability. also, the most recent 
-     * will be first on the list. 
+     * valid with a high probability. also, the most recent
+     * will be first on the list.
      * @param n the number of most recent snapshots
      * @return the last n snapshots (the number might be
      * less than n in case enough snapshots are not available).
@@ -156,7 +146,7 @@ public class FileSnap implements SnapShot {
             // from the valid snapshot and continue
             // until we find a valid one
             try {
-                if (Util.isValidSnapshot(f)) {
+                if (SnapStream.isValidSnapshot(f)) {
                     list.add(f);
                     count++;
                     if (count == n) {
@@ -221,18 +211,11 @@ public class FileSnap implements SnapShot {
     public synchronized void serialize(DataTree dt, Map<Long, Integer> sessions, File snapShot, boolean fsync)
             throws IOException {
         if (!close) {
-            try (CheckedOutputStream crcOut =
-                         new CheckedOutputStream(new BufferedOutputStream(fsync ? new AtomicFileOutputStream(snapShot) :
-                                                                                  new FileOutputStream(snapShot)),
-                                                 new Adler32())) {
-                //CheckedOutputStream cout = new CheckedOutputStream()
-                OutputArchive oa = BinaryOutputArchive.getArchive(crcOut);
+            try (CheckedOutputStream snapOS = SnapStream.getOutputStream(snapShot)) {
+                OutputArchive oa = BinaryOutputArchive.getArchive(snapOS);
                 FileHeader header = new FileHeader(SNAP_MAGIC, VERSION, dbId);
                 serialize(dt, sessions, oa, header);
-                long val = crcOut.getChecksum().getValue();
-                oa.writeLong(val, "val");
-                oa.writeString("/", "path");
-                crcOut.flush();
+                SnapStream.sealStream(snapOS, oa);
             }
         }
     }

zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/SnapStream.java

@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server.persistence;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.zip.Adler32;
+import java.util.zip.CheckedInputStream;
+import java.util.zip.CheckedOutputStream;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.jute.InputArchive;
+import org.apache.jute.OutputArchive;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xerial.snappy.SnappyCodec;
+import org.xerial.snappy.SnappyInputStream;
+import org.xerial.snappy.SnappyOutputStream;
+
+/**
+ * Represent the Stream used in serialize and deserialize the Snapshot.
+ */
+public class SnapStream {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SnapStream.class);
+
+    public static final String ZOOKEEPER_SHAPSHOT_STREAM_MODE =
+        "zookeeper.snapshot.compression.method";
+
+    private static StreamMode streamMode =
+        StreamMode.fromString(
+            System.getProperty(ZOOKEEPER_SHAPSHOT_STREAM_MODE,
+                  StreamMode.DEFAULT_MODE.getName()));
+
+    static {
+        LOG.info(ZOOKEEPER_SHAPSHOT_STREAM_MODE + "=" + streamMode);
+    }
+
+    public static enum StreamMode {
+        GZIP("gz"),
+        SNAPPY("snappy"),
+        CHECKED("");
+
+        public static final StreamMode DEFAULT_MODE = CHECKED;
+
+        private String name;
+
+        StreamMode(String name) {
+           this.name = name;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public String getFileExtension() {
+            return name.isEmpty() ? "" : "." + name;
+        }
+
+        public static StreamMode fromString(String name) {
+            for (StreamMode c : values()) {
+                if (c.getName().compareToIgnoreCase(name) == 0) {
+                    return c;
+                }
+            }
+            return DEFAULT_MODE;
+        }
+    }
+
+    /**
+     * Return the CheckedInputStream based on the extension of the fileName.
+     *
+     * @param fileName the file the InputStream read from
+     * @return the specific InputStream
+     * @throws IOException
+     */
+    public static CheckedInputStream getInputStream(File file) throws IOException {
+        FileInputStream fis = new FileInputStream(file);
+        InputStream is;
+        switch (getStreamMode(file.getName())) {
+            case GZIP:
+                is = new GZIPInputStream(fis);
+                break;
+            case SNAPPY:
+                is = new SnappyInputStream(fis);
+                break;
+            case CHECKED:
+            default:
+                is = new BufferedInputStream(fis);
+        }
+        return new CheckedInputStream(is, new Adler32());
+    }
+
+    /**
+     * Return the OutputStream based on predefined stream mode.
+     *
+     * @param fileName the file the OutputStream writes to
+     * @return the specific OutputStream
+     * @throws IOException
+     */
+    public static CheckedOutputStream getOutputStream(File file) throws IOException {
+        FileOutputStream fos = new FileOutputStream(file);
+        OutputStream os;
+        switch (streamMode) {
+            case GZIP:
+                os = new GZIPOutputStream(fos);
+                break;
+            case SNAPPY:
+                os = new SnappyOutputStream(fos);
+                break;
+            case CHECKED:
+            default:
+                os = new BufferedOutputStream(fos);
+        }
+        return new CheckedOutputStream(os, new Adler32());
+    }
+
+    /**
+     * Write specific seal to the OutputArchive and close the OutputStream.
+     * Currently, only CheckedOutputStream will write it's checkSum to the
+     * end of the stream.
+     *
+     */
+    public static void sealStream(CheckedOutputStream os, OutputArchive oa)
+            throws IOException {
+        long val = os.getChecksum().getValue();
+        oa.writeLong(val, "val");
+        oa.writeString("/", "path");
+    }
+
+    /**
+     * Verify the integrity of the seal, only CheckedInputStream will verify
+     * the checkSum of the content.
+     *
+     */
+    static void checkSealIntegrity(CheckedInputStream is, InputArchive ia)
+            throws IOException {
+        long checkSum = is.getChecksum().getValue();
+        long val = ia.readLong("val");
+        if (val != checkSum) {
+            throw new IOException("CRC corruption");
+        }
+    }
+
+    /**
+     * Verifies that the file is a valid snapshot. Snapshot may be invalid if
+     * it's incomplete as in a situation when the server dies while in the
+     * process of storing a snapshot. Any files that are improperly formated
+     * or corrupted are invalid. Any file that is not a snapshot is also an
+     * invalid snapshot.
+     *
+     * @param file file to verify
+     * @return true if the snapshot is valid
+     * @throws IOException
+     */
+    public static boolean isValidSnapshot(File file) throws IOException {
+        if (file == null || Util.getZxidFromName(file.getName(), FileSnap.SNAPSHOT_FILE_PREFIX) == -1) {
+            return false;
+        }
+
+        String fileName = file.getName();
+        if (Util.getZxidFromName(fileName, "snapshot") == -1) {
+            return false;
+        }
+
+        boolean isValid = false;
+        switch (getStreamMode(fileName)) {
+            case GZIP:
+                isValid = isValidGZipStream(file);
+                break;
+            case SNAPPY:
+                isValid = isValidSnappyStream(file);
+                break;
+            case CHECKED:
+            default:
+                isValid = isValidCheckedStream(file);
+        }
+        return isValid;
+    }
+
+    public static void setStreamMode(StreamMode mode) {
+        streamMode = mode;
+    }
+
+    public static StreamMode getStreamMode() {
+        return streamMode;
+    }
+
+    /**
+     * Detect the stream mode from file name extension
+     *
+     * @param fileName
+     * @return
+     */
+    public static StreamMode getStreamMode(String fileName) {
+        String[] splitSnapName = fileName.split("\\.");
+
+        // Use file extension to detect format
+        if (splitSnapName.length > 1) {
+            String mode = splitSnapName[splitSnapName.length - 1];
+            return StreamMode.fromString(mode);
+        }
+
+        return StreamMode.CHECKED;
+    }
+
+    /**
+     * Certify the GZip stream integrity by checking the header
+     * for the GZip magic string
+     *
+     * @param f file to verify
+     * @return true if it has the correct GZip magic string
+     * @throws IOException
+     */
+    private static boolean isValidGZipStream(File f) throws IOException {
+        byte[] byteArray = new byte[2];
+        try (FileInputStream fis = new FileInputStream(f)) {
+            if (2 != fis.read(byteArray, 0, 2)) {
+                LOG.error("Read incorrect number of bytes from " + f.getName());
+                return false;
+            }
+            ByteBuffer bb = ByteBuffer.wrap(byteArray);
+            byte[] magicHeader = new byte[2];
+            bb.get(magicHeader, 0, 2);
+            int magic = magicHeader[0] & 0xff | ((magicHeader[1] << 8) & 0xff00);
+            return magic == GZIPInputStream.GZIP_MAGIC;
+        } catch (FileNotFoundException e) {
+            LOG.error("Unable to open file " + f.getName() + " : ", e);
+            return false;
+        }
+    }
+
+    /**
+     * Certify the Snappy stream integrity by checking the header
+     * for the Snappy magic string
+     *
+     * @param f file to verify
+     * @return true if it has the correct Snappy magic string
+     * @throws IOException
+     */
+    private static boolean isValidSnappyStream(File f) throws IOException {
+        byte[] byteArray = new byte[SnappyCodec.MAGIC_LEN];
+        try (FileInputStream fis = new FileInputStream(f)) {
+            if (SnappyCodec.MAGIC_LEN != fis.read(byteArray, 0, SnappyCodec.MAGIC_LEN)) {
+                LOG.error("Read incorrect number of bytes from " + f.getName());
+                return false;
+            }
+            ByteBuffer bb = ByteBuffer.wrap(byteArray);
+            byte[] magicHeader = new byte[SnappyCodec.MAGIC_LEN];
+            bb.get(magicHeader, 0, SnappyCodec.MAGIC_LEN);
+            return Arrays.equals(magicHeader, SnappyCodec.getMagicHeader());
+        } catch (FileNotFoundException e) {
+            LOG.error("Unable to open file " + f.getName() + " : ", e);
+            return false;
+        }
+    }
+
+    /**
+     * Certify the Checked stream integrity by checking the header
+     * length and format
+     *
+     * @param f file to verify
+     * @return true if it has the correct header
+     * @throws IOException
+     */
+    private static boolean isValidCheckedStream(File f) throws IOException {
+        try (RandomAccessFile raf = new RandomAccessFile(f, "r")) {
+            // including the header and the last / bytes
+            // the snapshot should be at least 10 bytes
+            if (raf.length() < 10) {
+                return false;
+            }
+
+            raf.seek(raf.length() - 5);
+            byte bytes[] = new byte[5];
+            int readlen = 0;
+            int l;
+            while (readlen < 5 &&
+                (l = raf.read(bytes, readlen, bytes.length - readlen)) >= 0) {
+                readlen += l;
+            }
+            if (readlen != bytes.length) {
+                LOG.info("Invalid snapshot " + f.getName()
+                        + ". too short, len = " + readlen + " bytes");
+                return false;
+            }
+            ByteBuffer bb = ByteBuffer.wrap(bytes);
+            int len = bb.getInt();
+            byte b = bb.get();
+            if (len != 1 || b != '/') {
+                LOG.info("Invalid snapshot " + f.getName() + ". len = " + len
+                        + ", byte = " + (b & 0xff));
+                return false;
+            }
+        }
+
+        return true;
+    }
+}

zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/Util.java

@@ -22,7 +22,6 @@ import java.io.ByteArrayOutputStream;
 import java.io.EOFException;
 import java.io.File;
 import java.io.IOException;
-import java.io.RandomAccessFile;
 import java.io.Serializable;
 import java.net.URI;
 import java.nio.ByteBuffer;
@@ -42,7 +41,7 @@ import org.slf4j.LoggerFactory;
 import org.apache.zookeeper.txn.TxnHeader;
 
 /**
- * A collection of utility methods for dealing with file name parsing, 
+ * A collection of utility methods for dealing with file name parsing,
  * low level I/O file operations and marshalling/unmarshalling.
  */
 public class Util {
@@ -59,11 +58,11 @@ public class Util {
         return uri.replace('\\', '/');
     }
     /**
-     * Given two directory files the method returns a well-formed 
+     * Given two directory files the method returns a well-formed
      * logfile provider URI. This method is for backward compatibility with the
      * existing code that only supports logfile persistence and expects these two
      * parameters passed either on the command-line or in the configuration file.
-     * 
+     *
      * @param dataDir snapshot directory
      * @param dataLogDir transaction log directory
      * @return logfile provider URI
@@ -71,13 +70,13 @@ public class Util {
     public static URI makeFileLoggerURL(File dataDir, File dataLogDir){
         return URI.create(makeURIString(dataDir.getPath(),dataLogDir.getPath(),null));
     }
-    
+
     public static URI makeFileLoggerURL(File dataDir, File dataLogDir,String convPolicy){
         return URI.create(makeURIString(dataDir.getPath(),dataLogDir.getPath(),convPolicy));
     }
 
     /**
-     * Creates a valid transaction log file name. 
+     * Creates a valid transaction log file name.
      * 
      * @param zxid used as a file name suffix (extension)
      * @return file name
@@ -88,17 +87,18 @@ public class Util {
 
     /**
      * Creates a snapshot file name.
-     * 
+     *
      * @param zxid used as a suffix
      * @return file name
      */
     public static String makeSnapshotName(long zxid) {
-        return FileSnap.SNAPSHOT_FILE_PREFIX + "." + Long.toHexString(zxid);
+        return FileSnap.SNAPSHOT_FILE_PREFIX + "." + Long.toHexString(zxid)
+                + SnapStream.getStreamMode().getFileExtension();
     }
-    
+
     /**
      * Extracts snapshot directory property value from the container.
-     * 
+     *
      * @param props properties container
      * @return file representing the snapshot directory
      */
@@ -108,24 +108,24 @@ public class Util {
 
     /**
      * Extracts transaction log directory property value from the container.
-     * 
+     *
      * @param props properties container
      * @return file representing the txn log directory
      */
     public static File getLogDir(Properties props){
         return new File(props.getProperty(LOG_DIR));
     }
-    
+
     /**
      * Extracts the value of the dbFormatConversion attribute.
-     * 
+     *
      * @param props properties container
      * @return value of the dbFormatConversion attribute
      */
     public static String getFormatConversionPolicy(Properties props){
         return props.getProperty(DB_FORMAT_CONV);
     }
-   
+
     /**
      * Extracts zxid from the file name. The file name should have been created
      * using one of the {@link #makeLogName(long)} or {@link #makeSnapshotName(long)}.
@@ -137,7 +137,7 @@ public class Util {
     public static long getZxidFromName(String name, String prefix) {
         long zxid = -1;
         String nameParts[] = name.split("\\.");
-        if (nameParts.length == 2 && nameParts[0].equals(prefix)) {
+        if (nameParts.length >= 2 && nameParts[0].equals(prefix)) {
             try {
                 zxid = Long.parseLong(nameParts[1], 16);
             } catch (NumberFormatException e) {
@@ -146,53 +146,6 @@ public class Util {
         return zxid;
     }
 
-    /**
-     * Verifies that the file is a valid snapshot. Snapshot may be invalid if 
-     * it's incomplete as in a situation when the server dies while in the process
-     * of storing a snapshot. Any file that is not a snapshot is also 
-     * an invalid snapshot. 
-     * 
-     * @param f file to verify
-     * @return true if the snapshot is valid
-     * @throws IOException
-     */
-    public static boolean isValidSnapshot(File f) throws IOException {
-        if (f==null || Util.getZxidFromName(f.getName(), FileSnap.SNAPSHOT_FILE_PREFIX) == -1)
-            return false;
-
-        // Check for a valid snapshot
-        try (RandomAccessFile raf = new RandomAccessFile(f, "r")) {
-            // including the header and the last / bytes
-            // the snapshot should be at least 10 bytes
-            if (raf.length() < 10) {
-                return false;
-            }
-            raf.seek(raf.length() - 5);
-            byte bytes[] = new byte[5];
-            int readlen = 0;
-            int l;
-            while (readlen < 5 &&
-                    (l = raf.read(bytes, readlen, bytes.length - readlen)) >= 0) {
-                readlen += l;
-            }
-            if (readlen != bytes.length) {
-                LOG.info("Invalid snapshot " + f
-                        + " too short, len = " + readlen);
-                return false;
-            }
-            ByteBuffer bb = ByteBuffer.wrap(bytes);
-            int len = bb.getInt();
-            byte b = bb.get();
-            if (len != 1 || b != '/') {
-                LOG.info("Invalid snapshot " + f + " len = " + len
-                        + " byte = " + (b & 0xff));
-                return false;
-            }
-        }
-
-        return true;
-    }
-
     /**
      * Reads a transaction entry from the input archive.
      * @param ia archive to read from
@@ -215,11 +168,11 @@ public class Util {
         }catch(EOFException e){}
         return null;
     }
-    
+
 
     /**
      * Serializes transaction header and transaction data into a byte buffer.
-     *  
+     *
      * @param hdr transaction header
      * @param txn transaction data
      * @return serialized transaction record
@@ -239,7 +192,7 @@ public class Util {
 
     /**
      * Write the serialized transaction record to the output archive.
-     *  
+     *
      * @param oa output archive
      * @param bytes serialized transaction record
      * @throws IOException
@@ -249,8 +202,8 @@ public class Util {
         oa.writeBuffer(bytes, "txnEntry");
         oa.writeByte((byte) 0x42, "EOR"); // 'B'
     }
-    
-    
+
+
     /**
      * Compare file file names of form "prefix.version". Sort order result
      * returned in order of version.
@@ -274,7 +227,7 @@ public class Util {
             return ascending ? result : -result;
         }
     }
-    
+
     /**
      * Sort the list of files. Recency as determined by the version component
      * of the file name.

zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerTest.java

@@ -24,6 +24,7 @@ import java.util.List;
 
 import org.apache.zookeeper.ZKTestCase;
 import org.apache.zookeeper.server.persistence.FileTxnLog;
+import org.apache.zookeeper.server.persistence.SnapStream;
 import org.apache.zookeeper.server.persistence.Util;
 import org.apache.zookeeper.test.ClientBase;
 import org.junit.Assert;
@@ -125,7 +126,7 @@ public class ZooKeeperServerTest extends ZKTestCase {
             if (!f.exists()) {
                 f.createNewFile();
             }
-            Assert.assertFalse("Snapshot file size is greater than 9 bytes", Util.isValidSnapshot(f));
+            Assert.assertFalse("Snapshot file size is greater than 9 bytes", SnapStream.isValidSnapshot(f));
             Assert.assertTrue("Can't delete file", f.delete());
         } catch (IOException e) {
         } finally {

zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/SnapStreamTest.java

@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.CheckedInputStream;
+import java.util.zip.CheckedOutputStream;
+
+import org.apache.jute.BinaryInputArchive;
+import org.apache.jute.BinaryOutputArchive;
+import org.apache.jute.InputArchive;
+import org.apache.jute.OutputArchive;
+import org.apache.zookeeper.server.persistence.SnapStream.StreamMode;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.Assert;
+
+import static org.apache.zookeeper.test.ClientBase.createTmpDir;
+
+public class SnapStreamTest {
+
+    @After
+    public void tearDown() {
+        System.clearProperty(SnapStream.ZOOKEEPER_SHAPSHOT_STREAM_MODE);
+        SnapStream.setStreamMode(StreamMode.DEFAULT_MODE);
+    }
+
+    @Test
+    public void testStreamMode() {
+        Assert.assertEquals(StreamMode.CHECKED.getName(), "");
+        Assert.assertEquals(StreamMode.CHECKED.getFileExtension(), "");
+        Assert.assertEquals(StreamMode.CHECKED, StreamMode.fromString("name"));
+        Assert.assertEquals(StreamMode.GZIP.getName(), "gz");
+        Assert.assertEquals(StreamMode.GZIP.getFileExtension(), ".gz");
+        Assert.assertEquals(StreamMode.GZIP, StreamMode.fromString("gz"));
+        Assert.assertEquals(StreamMode.SNAPPY.getName(), "snappy");
+        Assert.assertEquals(StreamMode.SNAPPY.getFileExtension(), ".snappy");
+        Assert.assertEquals(StreamMode.SNAPPY, StreamMode.fromString("snappy"));
+    }
+
+    @Test
+    public void testGetStreamMode() {
+        Assert.assertEquals(
+            "expected to return un-compressed stream",
+            StreamMode.CHECKED,
+            SnapStream.getStreamMode("snapshot.180000e3a2")
+        );
+        Assert.assertEquals(
+            "expected to return snappy stream",
+            StreamMode.SNAPPY,
+            SnapStream.getStreamMode("snapshot.180000e3a2.snappy")
+        );
+        Assert.assertEquals(
+            "expected to return gzip stream",
+            StreamMode.GZIP,
+            SnapStream.getStreamMode("snapshot.180000e3a2.gz")
+        );
+    }
+
+    @Test
+    public void testSerializeDeserializeWithChecked() throws IOException {
+        testSerializeDeserialize(StreamMode.CHECKED, "");
+    }
+
+    @Test
+    public void testSerializeDeserializeWithSNAPPY() throws IOException {
+        testSerializeDeserialize(StreamMode.SNAPPY, ".snappy");
+    }
+
+    @Test
+    public void testSerializeDeserializeWithGZIP() throws IOException {
+        testSerializeDeserialize(StreamMode.GZIP, ".gz");
+    }
+
+    private void testSerializeDeserialize(StreamMode mode, String fileSuffix) throws IOException {
+        SnapStream.setStreamMode(mode);
+
+        // serialize with gzip stream
+        File tmpDir = createTmpDir();
+        File file = new File(tmpDir, "snapshot.180000e3a2" + fileSuffix);
+        CheckedOutputStream os = SnapStream.getOutputStream(file);
+        OutputArchive oa = BinaryOutputArchive.getArchive(os);
+        FileHeader header = new FileHeader(FileSnap.SNAP_MAGIC, 2, 1);
+        header.serialize(oa, "fileheader");
+        SnapStream.sealStream(os, oa);
+        os.flush();
+        os.close();
+
+        Assert.assertTrue(SnapStream.isValidSnapshot(file));
+
+        // deserialize with gzip stream
+        CheckedInputStream is = SnapStream.getInputStream(file);
+        InputArchive ia = BinaryInputArchive.getArchive(is);
+        FileHeader restoredHeader = new FileHeader();
+        restoredHeader.deserialize(ia, "fileheader");
+        Assert.assertEquals(
+                "magic not the same", restoredHeader, header);
+        SnapStream.checkSealIntegrity(is, ia);
+    }
+
+    private void checkInvalidSnapshot(String filename) throws IOException {
+        // set the output stream mode to CHECKED
+        SnapStream.setStreamMode(StreamMode.CHECKED);
+
+        // serialize to CHECKED file without magic header
+        File tmpDir = createTmpDir();
+        File file = new File(tmpDir, filename);
+        OutputStream os = SnapStream.getOutputStream(file);
+        os.write(1);
+        os.flush();
+        os.close();
+        Assert.assertFalse(SnapStream.isValidSnapshot(file));
+    }
+
+    @Test
+    public void testInvalidSnapshot() throws IOException {
+        Assert.assertFalse(SnapStream.isValidSnapshot(null));
+
+        checkInvalidSnapshot("snapshot.180000e3a2");
+        checkInvalidSnapshot("snapshot.180000e3a2.gz");
+        checkInvalidSnapshot("snapshot.180000e3a2.snappy");
+    }
+}