HADOOP-1089. Make the C++ version of write and read v-int agree with the Java versions. Contributed by Milind Bhandarkar.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@518338 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -62,6 +62,10 @@ Trunk (unreleased changes)
     restarted it consumes 80% CPU. (Dhruba Borthakur via
     tomwhite)
 
+19. HADOOP-1089.  Make the C++ version of write and read v-int
+    agree with the Java versions.  (Milind Bhandarkar via
+    tomwhite)
+
 
 Release 0.12.0 - 2007-03-02
 

src/c++/librecordio/Makefile

@@ -14,28 +14,30 @@
 # limitations under the License.
 #
 
+COPTS=-g3 -O0 -Wall
+
 all: librecordio.a test
 
 librecordio.a: recordio.o filestream.o binarchive.o csvarchive.o xmlarchive.o exception.o
 	ar cru librecordio.a recordio.o filestream.o binarchive.o csvarchive.o xmlarchive.o exception.o
 
 recordio.o: recordio.cc recordio.hh archive.hh
-	g++ -g3 -O0 -c -I${XERCESCROOT}/include -o recordio.o recordio.cc
+	g++ ${COPTS} -c -I${XERCESCROOT}/include -o recordio.o recordio.cc
 	
 filestream.o: filestream.cc recordio.hh filestream.hh
-	g++ -g3 -O0 -c -o filestream.o filestream.cc
+	g++ ${COPTS} -c -o filestream.o filestream.cc
 	
 binarchive.o: binarchive.cc recordio.hh binarchive.hh archive.hh
-	g++ -g3 -O0 -c -o binarchive.o binarchive.cc
+	g++ ${COPTS} -c -o binarchive.o binarchive.cc
 
 csvarchive.o: csvarchive.cc recordio.hh csvarchive.hh archive.hh
-	g++ -g3 -O0 -c -o csvarchive.o csvarchive.cc
+	g++ ${COPTS} -c -o csvarchive.o csvarchive.cc
 
 xmlarchive.o: xmlarchive.cc recordio.hh xmlarchive.hh archive.hh
-	g++ -g3 -O0 -c -I${XERCESCROOT}/include -o xmlarchive.o xmlarchive.cc
+	g++ ${COPTS} -c -I${XERCESCROOT}/include -o xmlarchive.o xmlarchive.cc
 		
 exception.o: exception.cc exception.hh
-	g++ -g3 -O0 -c -o exception.o exception.cc
+	g++ ${COPTS} -c -o exception.o exception.cc
 	
 recordio.cc: recordio.hh archive.hh exception.hh
 filestream.cc: recordio.hh filestream.hh 

src/c++/librecordio/archive.hh

@@ -72,6 +72,7 @@ public:
     }
     endMap(idx, tag);
   }
+  virtual ~IArchive() {}
 };
 
 class OArchive {
@@ -115,6 +116,7 @@ public:
     }
     endMap(v.size(), tag);
  }
+  virtual ~OArchive() {}
 };
 }; // end namespace hadoop
 #endif /*ARCHIVE_HH_*/

src/c++/librecordio/binarchive.cc

@@ -17,6 +17,7 @@
  */
 
 #include "binarchive.hh"
+#include <rpc/types.h>
 #include <rpc/xdr.h>
 
 
@@ -38,63 +39,6 @@ static void deserialize(T& t, InStream& stream)
   }
 }
 
-static void serializeInt(int32_t t, OutStream& stream)
-{
-  if (t >= -120 && t <= 127) {
-    int8_t b = t;
-    stream.write(&b, 1);
-    return;
-  }
-        
-  int8_t len = -120;
-  if (t < 0) {
-    t ^= 0xFFFFFFFF; // take one's complement
-    len = -124;
-  }
-        
-  uint32_t tmp = t;
-  while (tmp != 0) {
-    tmp = tmp >> 8;
-    len--;
-  }
-  
-  stream.write(&len, 1);      
-  len = (len < -124) ? -(len + 124) : -(len + 120);
-        
-  for (uint32_t idx = len; idx != 0; idx--) {
-    uint32_t shiftbits = (idx - 1) * 8;
-    uint32_t mask = 0xFF << shiftbits;
-    uint8_t b = (t & mask) >> shiftbits;
-    stream.write(&b, 1);
-  }
-}
-
-static void deserializeInt(int32_t& t, InStream& stream)
-{
-  int8_t b;
-  if (1 != stream.read(&b, 1)) {
-    throw new IOException("Error deserializing int");
-  }
-  if (b >= -120) {
-    t = b;
-    return;
-  }
-  bool isNegative = (b < -124);
-  b = isNegative ? -(b + 124) : -(b + 120);
-  uint8_t barr[b];
-  if (b != stream.read(barr, b)) {
-    throw new IOException("Error deserializing int");
-  }
-  t = 0;
-  for (int idx = 0; idx < b; idx++) {
-    t = t << 8;
-    t |= (barr[idx] & 0xFF);
-  }
-  if (isNegative) {
-    t ^= 0xFFFFFFFF;
-  }
-}
-
 static void serializeLong(int64_t t, OutStream& stream)
 {
   if (t >= -112 && t <= 127) {
@@ -105,7 +49,7 @@ static void serializeLong(int64_t t, OutStream& stream)
         
   int8_t len = -112;
   if (t < 0) {
-    t &= 0xFFFFFFFFFFFFFFFFLL; // take one's complement
+    t ^= 0xFFFFFFFFFFFFFFFFLL; // take one's complement
     len = -120;
   }
         
@@ -149,10 +93,23 @@ static void deserializeLong(int64_t& t, InStream& stream)
     t |= (barr[idx] & 0xFF);
   }
   if (isNegative) {
-    t ^= 0xFFFFFFFFFFFFFFFFL;
+    t ^= 0xFFFFFFFFFFFFFFFFLL;
   }
 }
 
+static void serializeInt(int32_t t, OutStream& stream)
+{
+  int64_t longVal = t;
+  ::serializeLong(longVal, stream);
+}
+
+static void deserializeInt(int32_t& t, InStream& stream)
+{
+  int64_t longVal;
+  ::deserializeLong(longVal, stream);
+  t = longVal;
+}
+
 static void serializeFloat(float t, OutStream& stream)
 {
   char buf[sizeof(float)];
@@ -223,7 +180,9 @@ void hadoop::IBinArchive::deserialize(bool& t, const char* tag)
 
 void hadoop::IBinArchive::deserialize(int32_t& t, const char* tag)
 {
-  ::deserializeInt(t, stream);
+  int64_t longVal = 0LL;
+  ::deserializeLong(longVal, stream);
+  t = longVal;
 }
 
 void hadoop::IBinArchive::deserialize(int64_t& t, const char* tag)
@@ -302,7 +261,8 @@ void hadoop::OBinArchive::serialize(bool t, const char* tag)
 
 void hadoop::OBinArchive::serialize(int32_t t, const char* tag)
 {
-  ::serializeInt(t, stream);
+  int64_t longVal = t;
+  ::serializeLong(longVal, stream);
 }
 
 void hadoop::OBinArchive::serialize(int64_t t, const char* tag)

src/c++/librecordio/csvarchive.cc

@@ -114,7 +114,7 @@ void hadoop::ICsvArchive::deserialize(std::string& t, size_t& len, const char* t
   if (len%2 == 1) { // len is guaranteed to be even
     throw new IOException("Errror deserializing buffer.");
   }
-  len >> 1;
+  len = len >> 1;
   for (size_t idx = 0; idx < len; idx++) {
     char buf[3];
     buf[0] = s[2*idx];
@@ -298,7 +298,7 @@ void hadoop::OCsvArchive::serialize(const std::string& t, size_t len, const char
 {
   printCommaUnlessFirst();
   stream.write("#",1);
-  for(int idx = 0; idx < len; idx++) {
+  for(size_t idx = 0; idx < len; idx++) {
     uint8_t b = t[idx];
     char sval[3];
     sprintf(sval,"%2x",b);

src/c++/librecordio/exception.cc

@@ -17,7 +17,9 @@
  */
 
 #include "exception.hh"
+#ifdef USE_EXECINFO
 #include <execinfo.h>
+#endif
 
 #include <errno.h>
 #include <sstream>
@@ -42,7 +44,11 @@ namespace hadoop {
                           mReason(reason)
                           
   {
+#ifdef USE_EXECINFO
     mCalls = backtrace(mCallStack, sMaxCallStackDepth);
+#else
+    mCalls = 0;
+#endif
   }
 
   /**
@@ -81,7 +87,9 @@ namespace hadoop {
     if (mLocation.size() != 0) {
       stream << "  thrown at " << mLocation << "\n";
     }
+#ifdef USE_EXECINFO
     printCallStack(stream);
+#endif
     if (mReason) {
       stream << "caused by: ";
       mReason->print(stream);
@@ -98,6 +106,7 @@ namespace hadoop {
     return stream.str();
 }
 
+#ifdef USE_EXECINFO
   /**
    * Print the call stack where the exception was created.
    */
@@ -114,6 +123,7 @@ namespace hadoop {
       }
       free(symbols);
   }
+#endif
 
   const char* Exception::getTypename() const {
     return "Exception";

src/c++/librecordio/exception.hh

@@ -68,10 +68,12 @@ namespace hadoop {
      */
     virtual std::string toString() const;
 
+#ifdef USE_EXECINFO
     /**
      * Print the call stack where the exception was created.
      */
     virtual void printCallStack(std::ostream& stream=std::cerr) const;
+#endif
 
     const std::string& getMessage() const {
       return mMessage;

src/c++/librecordio/recordio.hh

@@ -33,11 +33,13 @@ namespace hadoop {
 class InStream {
 public:
   virtual ssize_t read(void *buf, size_t buflen) = 0;
+  virtual ~InStream() {}
 };
 
 class OutStream {
 public:
   virtual ssize_t write(const void *buf, size_t len) = 0;
+  virtual ~OutStream() {}
 };
 
 class IArchive;
@@ -45,11 +47,11 @@ class OArchive;
 
 class Record {
 public:
-  virtual bool validate() const = 0;
   virtual void serialize(OArchive& archive, const char* tag) const = 0;
   virtual void deserialize(IArchive& archive, const char* tag) = 0;
   virtual const std::string& type() const = 0;
   virtual const std::string& signature() const = 0;
+  virtual ~Record() {}
 };
 
 enum RecFormat { kBinary, kXML, kCSV };

src/c++/librecordio/test/Makefile

@@ -14,23 +14,25 @@
 # limitations under the License.
 #
 
+COPTS=-g3 -O0 -Wall
+
 all: test testFromJava
 
 test: test.o test.jr.o
 	g++ -g3 -O0 -o test test.o test.jr.o -L.. -L${XERCESCROOT}/lib -lrecordio -lxerces-c
 	
 test.o: test.cc
-	g++ -g3 -O0 -c -I.. -o test.o test.cc
+	g++ ${COPTS} -c -I.. -o test.o test.cc
 
 testFromJava: testFromJava.o test.jr.o
 	g++ -g3 -O0 -o testFromJava testFromJava.o test.jr.o -L.. -L${XERCESCROOT}/lib -lrecordio -lxerces-c
 	
 testFromJava.o: testFromJava.cc
-	g++ -g3 -O0 -c -I.. -o testFromJava.o testFromJava.cc
+	g++ ${COPTS} -c -I.. -o testFromJava.o testFromJava.cc
 
 
 test.jr.o: test.jr.cc
-	g++ -g3 -O0 -c -I.. -o test.jr.o test.jr.cc
+	g++ ${COPTS} -c -I.. -o test.jr.o test.jr.cc
 
 %.jr.cc %.jr.hh: %.jr
 	${HADOOP_HOME}/bin/rcc --language c++ $<

src/c++/librecordio/test/test.cc

@@ -34,9 +34,6 @@ int main()
     r1.setLongVal(0x5a5a5a5a5a5aLL);
     std::string& s = r1.getStringVal();
     s = "random text";
-    std::string& buf = r1.getBufferVal();
-    std::vector<std::string>& v = r1.getVectorVal();
-    std::map<std::string,std::string>& m = r1.getMapVal();
     writer.write(r1);
     ostream.close();
     hadoop::FileInStream istream;
@@ -62,9 +59,6 @@ int main()
     r1.setLongVal(0x5a5a5a5a5a5aLL);
     std::string& s = r1.getStringVal();
     s = "random text";
-    std::string& buf = r1.getBufferVal();
-    std::vector<std::string>& v = r1.getVectorVal();
-    std::map<std::string,std::string>& m = r1.getMapVal();
     writer.write(r1);
     ostream.close();
     hadoop::FileInStream istream;
@@ -90,9 +84,6 @@ int main()
     r1.setLongVal(0x5a5a5a5a5a5aLL);
     std::string& s = r1.getStringVal();
     s = "random text";
-    std::string& buf = r1.getBufferVal();
-    std::vector<std::string>& v = r1.getVectorVal();
-    std::map<std::string,std::string>& m = r1.getMapVal();
     writer.write(r1);
     ostream.close();
     hadoop::FileInStream istream;

src/c++/librecordio/test/testFromJava.cc

@@ -30,9 +30,6 @@ int main()
   r1.setLongVal(0x5a5a5a5a5a5aLL);
   std::string& s = r1.getStringVal();
   s = "random text";
-  std::string& buf = r1.getBufferVal();
-  std::vector<std::string>& v = r1.getVectorVal();
-  std::map<std::string,std::string>& m = r1.getMapVal();
   {
     hadoop::FileInStream istream;
     istream.open("/tmp/hadooptemp.dat");

src/c++/librecordio/xmlarchive.cc

@@ -164,7 +164,7 @@ static std::string fromXMLBuffer(std::string s, size_t& len)
   if (len%2 == 1) { // len is guaranteed to be even
     throw new IOException("Errror deserializing buffer.");
   }
-  len >> 1;
+  len = len >> 1;
   std::string t;
   for (size_t idx = 0; idx < len; idx++) {
     char buf[3];

src/java/org/apache/hadoop/record/compiler/CppGenerator.java

@@ -46,20 +46,21 @@ class CppGenerator extends CodeGenerator {
     FileWriter cc = new FileWriter(name+".cc");
     FileWriter hh = new FileWriter(name+".hh");
     
-    hh.write("#ifndef __"+name.toUpperCase().replace('.','_')+"__\n");
-    hh.write("#define __"+name.toUpperCase().replace('.','_')+"__\n");
+    String fileName = (new File(name)).getName();
+    hh.write("#ifndef __"+fileName.toUpperCase().replace('.','_')+"__\n");
+    hh.write("#define __"+fileName.toUpperCase().replace('.','_')+"__\n");
     hh.write("#include \"recordio.hh\"\n");
     for (Iterator<JFile> iter = ilist.iterator(); iter.hasNext();) {
       hh.write("#include \""+iter.next().getName()+".hh\"\n");
     }
     
-    cc.write("#include \""+name+".hh\"\n");
+    cc.write("#include \""+fileName+".hh\"\n");
     
     for (Iterator<JRecord> iter = rlist.iterator(); iter.hasNext();) {
       iter.next().genCppCode(hh, cc, options);
     }
     
-    hh.write("#endif //"+name.toUpperCase().replace('.','_')+"__\n");
+    hh.write("#endif //"+fileName.toUpperCase().replace('.','_')+"__\n");
     
     hh.close();
     cc.close();

src/java/org/apache/hadoop/record/compiler/JMap.java

@@ -161,7 +161,7 @@ public class JMap extends JCompType {
   /** Creates a new instance of JMap */
   public JMap(JType t1, JType t2) {
     setJavaType(new JavaMap(t1.getJavaType(), t2.getJavaType()));
-    setCppType(new CppType(" ::std::map<"+t1.getCppType().getType()+","+
+    setCppType(new CppCompType(" ::std::map<"+t1.getCppType().getType()+","+
         t2.getCppType().getType()+">"));
     setCType(new CType());
     keyType = t1;