From f830367d9d5db789d235c4448266f2acbbebe40f Mon Sep 17 00:00:00 2001 From: Randolph Voorhies Date: Mon, 8 Jul 2013 17:14:40 -0700 Subject: [PATCH] Writing arrays now works. Reading is next --- include/cereal/archives/json.hpp | 151 +++++++++++++++--- .../cereal/external/rapidjson/prettywriter.h | 4 +- include/cereal/external/rapidjson/writer.h | 71 ++++++-- sandbox_json.cpp | 59 +++---- unittests.cpp | 132 ++++++++++++++- 5 files changed, 342 insertions(+), 75 deletions(-) diff --git a/include/cereal/archives/json.hpp b/include/cereal/archives/json.hpp index 2ccc359c..8fe257a6 100644 --- a/include/cereal/archives/json.hpp +++ b/include/cereal/archives/json.hpp @@ -53,20 +53,23 @@ namespace cereal \ingroup Archives */ class JSONOutputArchive : public OutputArchive { + enum class NodeType { StartObject, InObject, StartArray, InArray }; + typedef rapidjson::GenericWriteStream WriteStream; typedef rapidjson::PrettyWriter JSONWriter; public: //! Construct, outputting to the provided stream /*! @param stream The stream to output to. Can be a stringstream, a file stream, or - even cout! */ - JSONOutputArchive(std::ostream & stream) : + even cout! + @param precision The precision for floating point output */ + JSONOutputArchive(std::ostream & stream, int precision = 20) : OutputArchive(this), itsWriteStream(stream), - itsWriter(itsWriteStream) + itsWriter(itsWriteStream, precision) { itsNameCounter.push(0); - itsWriter.StartObject(); + itsNodeStack.push(NodeType::StartObject); } //! Destructor, flushes the JSON @@ -84,9 +87,34 @@ namespace cereal void saveValue(std::string const & s) { itsWriter.String(s.c_str(), s.size()); } void saveValue(char const * s) { itsWriter.String(s); } + //! Save exotic arithmetic types as binary + template + typename std::enable_if::value && + (sizeof(T) >= sizeof(long double) || sizeof(T) >= sizeof(long long)), void>::type + saveValue(T const & t) + { + auto base64string = base64::encode( reinterpret_cast( &t ), sizeof(T) ); + saveValue( base64string ); + } + //! Write the name of the upcoming node void writeName() { + NodeType const & nodeType = itsNodeStack.top(); + + if(nodeType == NodeType::StartArray) + { + itsWriter.StartArray(); + itsNodeStack.top() = NodeType::InArray; + } + else if(nodeType == NodeType::StartObject) + { + itsNodeStack.top() = NodeType::InObject; + itsWriter.StartObject(); + } + + if(nodeType == NodeType::InArray) return; + if(itsNextName == nullptr) { std::string name = "value" + std::to_string( itsNameCounter.top()++ ) + "\0"; @@ -97,30 +125,42 @@ namespace cereal saveValue(itsNextName); itsNextName = nullptr; } + } - //! Creates a new node that is a child of the node at the top of the stack - /*! Nodes will be given a name that has either been pre-set by a name value pair, - or generated based upon a counter unique to the parent node. - - The node will then be pushed onto the node stack. */ void startNode() { writeName(); - - itsWriter.StartObject(); - + itsNodeStack.push(NodeType::StartObject); itsNameCounter.push(0); } //! Designates the most recently added node as finished void finishNode() { - itsWriter.EndObject(); + switch(itsNodeStack.top()) + { + case NodeType::StartArray: + itsWriter.StartArray(); + case NodeType::InArray: + itsWriter.EndArray(); + break; + case NodeType::StartObject: + itsWriter.StartObject(); + case NodeType::InObject: + itsWriter.EndObject(); + break; + } + itsNodeStack.pop(); itsNameCounter.pop(); } + void makeArray() + { + itsNodeStack.top() = NodeType::StartArray; + } + //! Sets the name for the next node created with startNode void setNextName( const char * name ) { @@ -151,6 +191,7 @@ namespace cereal bool itsOutputType; //!< Controls whether type information is printed char const * itsNextName; //!< The next name std::stack itsNameCounter; //!< Counter for creating unique names for unnamed nodes + std::stack itsNodeStack; }; // JSONOutputArchive // ###################################################################### @@ -192,14 +233,68 @@ namespace cereal } void loadValue(bool & val) { val = itsValueStack.top()->value.GetBool(); ++itsValueStack.top(); } - void loadValue(int & val) { val = itsValueStack.top()->value.GetInt(); ++itsValueStack.top(); } - void loadValue(unsigned & val) { val = itsValueStack.top()->value.GetUint(); ++itsValueStack.top(); } + + template + typename std::enable_if::value && sizeof(T) < sizeof(int64_t), void>::type + loadValue(T & val) + { + val = itsValueStack.top()->value.GetInt(); + ++itsValueStack.top(); + } + + template + typename std::enable_if<(std::is_unsigned::value && sizeof(T) < sizeof(uint64_t)) && + !std::is_same::value, void>::type + loadValue(T & val) + { + val = itsValueStack.top()->value.GetUint(); + ++itsValueStack.top(); + } + void loadValue(int64_t & val) { val = itsValueStack.top()->value.GetInt64(); ++itsValueStack.top(); } void loadValue(uint64_t & val) { val = itsValueStack.top()->value.GetUint64(); ++itsValueStack.top(); } void loadValue(float & val) { val = itsValueStack.top()->value.GetDouble(); ++itsValueStack.top(); } void loadValue(double & val) { val = itsValueStack.top()->value.GetDouble(); ++itsValueStack.top(); } void loadValue(std::string & val) { val = itsValueStack.top()->value.GetString(); ++itsValueStack.top(); } + template + typename std::enable_if::value && + (sizeof(T) >= sizeof(long double) || sizeof(T) >= sizeof(long long)), void>::type + loadValue(T & val) + { + std::string encoded; + loadValue( encoded ); + auto decoded = base64::decode( encoded ); + + if( sizeof(T) != decoded.size() ) + throw Exception("Decoded binary data size does not match specified size"); + + std::memcpy( &val, decoded.data(), decoded.size() ); + } + + //! Loads some binary data, encoded as a base64 string, with an optional name + void loadBinaryValue( void * data, size_t size, char const * name = nullptr) + { + startNode(); + + std::string encoded; + loadValue( encoded ); + auto decoded = base64::decode( encoded ); + + if( size != decoded.size() ) + throw Exception("Decoded binary data size does not match specified size"); + + std::memcpy( data, decoded.data(), decoded.size() ); + + finishNode(); + }; + + void loadSize(size_t & size) + { + size = itsValueStack.top()->value.Size(); + } + + private: char const * itsNextName; ReadStream itsReadStream; //!< Rapidjson write stream @@ -240,12 +335,12 @@ namespace cereal //! Prologue for SizeTags for JSON archives /*! SizeTags are strictly ignored for JSON */ template - void prologue( JSONOutputArchive &, SizeTag const & ) - { } + void prologue( JSONOutputArchive & ar, SizeTag const & ) + { ar.makeArray(); } //! Prologue for SizeTags for JSON archives template - void prologue( JSONInputArchive &, SizeTag const & ) + void prologue( JSONInputArchive & ar, SizeTag const & ) { } // ###################################################################### @@ -365,12 +460,6 @@ namespace cereal ar( t.value ); } - //! Serializing SizeTags to JSON - template inline - CEREAL_ARCHIVE_RESTRICT(JSONInputArchive, JSONOutputArchive) - serialize( Archive &, SizeTag & ) - { } - //! Saving for arithmetic to JSON template inline typename std::enable_if::value, void>::type @@ -400,6 +489,20 @@ namespace cereal { ar.loadValue( str ); } + + // ###################################################################### + //! Saving SizeTags to JSON + template inline + void save( JSONOutputArchive &, SizeTag const & ) + { } + + //! Loading SizeTags from JSON + template inline + void load( JSONInputArchive & ar, SizeTag & st ) + { + ar.loadSize( st.size ); + } + } // namespace cereal // register archives for polymorphic support diff --git a/include/cereal/external/rapidjson/prettywriter.h b/include/cereal/external/rapidjson/prettywriter.h index 90596fc1..eccd2563 100644 --- a/include/cereal/external/rapidjson/prettywriter.h +++ b/include/cereal/external/rapidjson/prettywriter.h @@ -22,8 +22,8 @@ public: \param allocator User supplied allocator. If it is null, it will create a private one. \param levelDepth Initial capacity of */ - PrettyWriter(Stream& stream, Allocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(stream, allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} + PrettyWriter(Stream& stream, int precision = 20, Allocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(stream, precision, allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} //! Set custom indentation. /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\t', '\n', '\r'). diff --git a/include/cereal/external/rapidjson/writer.h b/include/cereal/external/rapidjson/writer.h index a83c0316..47744ebe 100644 --- a/include/cereal/external/rapidjson/writer.h +++ b/include/cereal/external/rapidjson/writer.h @@ -34,18 +34,31 @@ class Writer { public: typedef typename Encoding::Ch Ch; - Writer(Stream& stream, Allocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : - stream_(stream), level_stack_(allocator, levelDepth * sizeof(Level)) {} + Writer(Stream& stream, int precision = 20, Allocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : + stream_(stream), level_stack_(allocator, levelDepth * sizeof(Level)) + { + (void) snprintf(double_format, sizeof(double_format), "%%0.%dg", precision); + (void) snprintf(long_double_format, sizeof(long_double_format), "%%0.%dLg", precision); + } + +protected: + char double_format[32]; + char long_double_format[32]; +public: //@name Implementation of Handler //@{ - Writer& Null() { Prefix(kNullType); WriteNull(); return *this; } - Writer& Bool(bool b) { Prefix(b ? kTrueType : kFalseType); WriteBool(b); return *this; } - Writer& Int(int i) { Prefix(kNumberType); WriteInt(i); return *this; } - Writer& Uint(unsigned u) { Prefix(kNumberType); WriteUint(u); return *this; } - Writer& Int64(int64_t i64) { Prefix(kNumberType); WriteInt64(i64); return *this; } - Writer& Uint64(uint64_t u64) { Prefix(kNumberType); WriteUint64(u64); return *this; } - Writer& Double(double d) { Prefix(kNumberType); WriteDouble(d); return *this; } + + Writer& Null() { Prefix(kNullType); WriteNull(); return *this; } + Writer& Bool(bool b) { Prefix(b ? kTrueType : kFalseType); WriteBool(b); return *this; } + Writer& Int(int i) { Prefix(kNumberType); WriteInt(i); return *this; } + Writer& Uint(unsigned u) { Prefix(kNumberType); WriteUint(u); return *this; } + Writer& Int64(int64_t i64) { Prefix(kNumberType); WriteInt64(i64); return *this; } + Writer& Uint64(uint64_t u64) { Prefix(kNumberType); WriteUint64(u64); return *this; } + Writer& Double(double d) { Prefix(kNumberType); WriteDouble(d); return *this; } + Writer& LongDouble(long double d) { Prefix(kNumberType); WriteLongDouble(d); return *this; } + Writer& LongLong(long long d) { Prefix(kNumberType); WriteLongLong(d); return *this; } + Writer& ULongLong(unsigned long long d) { Prefix(kNumberType); WriteULongLong(d); return *this; } Writer& String(const Ch* str, SizeType length, bool copy = false) { (void)copy; @@ -171,9 +184,45 @@ protected: void WriteDouble(double d) { char buffer[100]; #if _MSC_VER - int ret = sprintf_s(buffer, sizeof(buffer), "%g", d); + int ret = sprintf_s(buffer, sizeof(buffer), double_format, d); #else - int ret = snprintf(buffer, sizeof(buffer), "%g", d); + int ret = snprintf(buffer, sizeof(buffer), double_format, d); +#endif + RAPIDJSON_ASSERT(ret >= 1); + for (int i = 0; i < ret; i++) + stream_.Put(buffer[i]); + } + + void WriteLongDouble(long double d) { + char buffer[256]; +#if _MSC_VER + int ret = sprintf_s(buffer, sizeof(buffer), long_double_format, d); +#else + int ret = snprintf(buffer, sizeof(buffer), long_double_format, d); +#endif + RAPIDJSON_ASSERT(ret >= 1); + for (int i = 0; i < ret; i++) + stream_.Put(buffer[i]); + } + + void WriteLongLong(long long d) { + char buffer[256]; +#if _MSC_VER + int ret = sprintf_s(buffer, sizeof(buffer), "%lld", d); +#else + int ret = snprintf(buffer, sizeof(buffer), "%lld", d); +#endif + RAPIDJSON_ASSERT(ret >= 1); + for (int i = 0; i < ret; i++) + stream_.Put(buffer[i]); + } + + void WriteULongLong(unsigned long long d) { + char buffer[256]; +#if _MSC_VER + int ret = sprintf_s(buffer, sizeof(buffer), "%llu", d); +#else + int ret = snprintf(buffer, sizeof(buffer), "%llu", d); #endif RAPIDJSON_ASSERT(ret >= 1); for (int i = 0; i < ret; i++) diff --git a/sandbox_json.cpp b/sandbox_json.cpp index 4683888f..6c107781 100644 --- a/sandbox_json.cpp +++ b/sandbox_json.cpp @@ -216,6 +216,20 @@ struct Fixture } }; +struct AAA +{ + int one = 1, two = 2; + + std::vector> three = {{1,2,3}, {4,5,6}, {}}; + + template + void serialize(Archive & ar) + { + ar( CEREAL_NVP(one), CEREAL_NVP(two) ); + ar( CEREAL_NVP(three) ); + } +}; + // ###################################################################### int main() { @@ -225,15 +239,12 @@ int main() std::ofstream os("file.json"); cereal::JSONOutputArchive oar( os ); - Fixture f1; + //std::vector x = {1, 2, 3, 4, 5}; + //std::vector x(3); + std::array x; - int x = 10; - double y = 99; + oar( CEREAL_NVP(x) ); - oar(CEREAL_NVP(f1)); - - oar(CEREAL_NVP(x)); - oar(CEREAL_NVP(y)); } { @@ -242,40 +253,14 @@ int main() std::cout << "---------------------" << std::endl << str << std::endl << "---------------------" << std::endl; } - { - std::ifstream is("file.json"); - cereal::JSONInputArchive iar( is ); - - Fixture f1; - - int x; - double y; - iar(CEREAL_NVP(f1)); - iar(CEREAL_NVP(x)); - iar(CEREAL_NVP(y)); - - std::cout << x << " " << y << std::endl; - - std::cout << f1.f1.a << std::endl; - } - //{ // std::ifstream is("file.json"); - // rapidjson::GenericReadStream itsReadStream(is); - // rapidjson::Document itsDocument; - // itsDocument.ParseStream<0>(itsReadStream); - // - // //FILE * f = std::fopen("file.json", "r"); - // //rapidjson::FileStream filestream(f); - // //rapidjson::Document itsDocument; - // //itsDocument.ParseStream<0>(filestream); + // cereal::JSONInputArchive iar( is ); - // std::cout << itsDocument.IsObject() << std::endl; - // std::cout << itsDocument.HasMember("x") << std::endl; + // std::vector x; + + // iar( CEREAL_NVP(x) ); - // auto it = itsDocument.MemberBegin(); - // std::cout << it->name.GetString() << std::endl; - // it->value.GetInt(); //} return 0; diff --git a/unittests.cpp b/unittests.cpp index 0965c4d5..ef1bdf35 100644 --- a/unittests.cpp +++ b/unittests.cpp @@ -46,6 +46,7 @@ #include #include +#include #include #include @@ -275,7 +276,7 @@ void test_pod() BOOST_CHECK_EQUAL(i_int32 , o_int32); BOOST_CHECK_EQUAL(i_uint64 , o_uint64); BOOST_CHECK_EQUAL(i_int64 , o_int64); - BOOST_CHECK_CLOSE(i_float , o_float, 1e-5); + BOOST_CHECK_CLOSE(i_float , o_float, 1e-5); BOOST_CHECK_CLOSE(i_double , o_double, 1e-5); } } @@ -290,6 +291,11 @@ BOOST_AUTO_TEST_CASE( xml_pod ) test_pod(); } +BOOST_AUTO_TEST_CASE( json_pod ) +{ + test_pod(); +} + // ###################################################################### template void test_structs() @@ -338,6 +344,11 @@ BOOST_AUTO_TEST_CASE( xml_structs ) test_structs(); } +BOOST_AUTO_TEST_CASE( json_structs ) +{ + test_structs(); +} + // ###################################################################### template void test_array() @@ -413,6 +424,11 @@ BOOST_AUTO_TEST_CASE( xml_array ) test_array(); } +BOOST_AUTO_TEST_CASE( json_array ) +{ + test_array(); +} + // ###################################################################### template void test_deque() @@ -494,6 +510,11 @@ BOOST_AUTO_TEST_CASE( xml_dequeue ) test_deque(); } +BOOST_AUTO_TEST_CASE( json_dequeue ) +{ + test_deque(); +} + // ###################################################################### template void test_forward_list() @@ -569,6 +590,11 @@ BOOST_AUTO_TEST_CASE( xml_forward_list ) test_forward_list(); } +BOOST_AUTO_TEST_CASE( json_forward_list ) +{ + test_forward_list(); +} + // ###################################################################### template void test_list() @@ -644,6 +670,11 @@ BOOST_AUTO_TEST_CASE( xml_list ) test_list(); } +BOOST_AUTO_TEST_CASE( json_list ) +{ + test_list(); +} + // ###################################################################### template void test_map() @@ -739,6 +770,11 @@ BOOST_AUTO_TEST_CASE( xml_map ) test_map(); } +BOOST_AUTO_TEST_CASE( json_map ) +{ + test_map(); +} + // ###################################################################### template void test_multimap() @@ -848,6 +884,11 @@ BOOST_AUTO_TEST_CASE( xml_multimap ) test_multimap(); } +BOOST_AUTO_TEST_CASE( json_multimap ) +{ + test_multimap(); +} + // ###################################################################### template void test_memory() @@ -911,6 +952,11 @@ BOOST_AUTO_TEST_CASE( xml_memory ) test_memory(); } +BOOST_AUTO_TEST_CASE( json_memory ) +{ + test_memory(); +} + // ###################################################################### template void test_queue() @@ -998,6 +1044,11 @@ BOOST_AUTO_TEST_CASE( xml_queue ) test_queue(); } +BOOST_AUTO_TEST_CASE( json_queue ) +{ + test_queue(); +} + // ###################################################################### template void test_priority_queue() @@ -1085,6 +1136,11 @@ BOOST_AUTO_TEST_CASE( xml_priority_queue ) test_priority_queue(); } +BOOST_AUTO_TEST_CASE( json_priority_queue ) +{ + test_priority_queue(); +} + // ###################################################################### template void test_set() @@ -1160,6 +1216,11 @@ BOOST_AUTO_TEST_CASE( xml_set ) test_set(); } +BOOST_AUTO_TEST_CASE( json_set ) +{ + test_set(); +} + // ###################################################################### template void test_multiset() @@ -1274,6 +1335,11 @@ BOOST_AUTO_TEST_CASE( xml_multiset ) test_multiset(); } +BOOST_AUTO_TEST_CASE( json_multiset ) +{ + test_multiset(); +} + // ###################################################################### template void test_stack() @@ -1361,6 +1427,11 @@ BOOST_AUTO_TEST_CASE( xml_stack ) test_stack(); } +BOOST_AUTO_TEST_CASE( json_stack ) +{ + test_stack(); +} + // ###################################################################### template void test_string() @@ -1521,6 +1592,11 @@ BOOST_AUTO_TEST_CASE( xml_unordered_map ) test_unordered_map(); } +BOOST_AUTO_TEST_CASE( json_unordered_map ) +{ + test_unordered_map(); +} + // ###################################################################### template void test_unordered_multimap() @@ -1656,6 +1732,11 @@ BOOST_AUTO_TEST_CASE( xml_unordered_multimap ) test_unordered_multimap(); } +BOOST_AUTO_TEST_CASE( json_unordered_multimap ) +{ + test_unordered_multimap(); +} + // ###################################################################### template void test_unordered_set() @@ -1750,6 +1831,11 @@ BOOST_AUTO_TEST_CASE( xml_unordered_set ) test_unordered_set(); } +BOOST_AUTO_TEST_CASE( json_unordered_set ) +{ + test_unordered_set(); +} + // ###################################################################### template void test_unordered_multiset() @@ -1864,6 +1950,11 @@ BOOST_AUTO_TEST_CASE( xml_unordered_multiset ) test_unordered_multiset(); } +BOOST_AUTO_TEST_CASE( json_unordered_multiset ) +{ + test_unordered_multiset(); +} + // ###################################################################### template void test_vector() @@ -1945,6 +2036,11 @@ BOOST_AUTO_TEST_CASE( xml_vector ) test_vector(); } +BOOST_AUTO_TEST_CASE( json_vector ) +{ + test_vector(); +} + // ###################################################################### template void test_pair() @@ -2016,6 +2112,10 @@ BOOST_AUTO_TEST_CASE( xml_pair ) { test_pair(); } +BOOST_AUTO_TEST_CASE( json_pair ) +{ + test_pair(); +} // ###################################################################### template @@ -2092,6 +2192,11 @@ BOOST_AUTO_TEST_CASE( xml_tuple ) test_tuple(); } +BOOST_AUTO_TEST_CASE( json_tuple ) +{ + test_tuple(); +} + // ###################################################################### template void test_complex() @@ -2148,6 +2253,11 @@ BOOST_AUTO_TEST_CASE( xml_complex ) test_complex(); } +BOOST_AUTO_TEST_CASE( json_complex ) +{ + test_complex(); +} + // ###################################################################### template void test_bitset() @@ -2203,6 +2313,11 @@ BOOST_AUTO_TEST_CASE( xml_bitset ) test_bitset(); } +BOOST_AUTO_TEST_CASE( json_bitset ) +{ + test_bitset(); +} + // ###################################################################### template void test_chrono() @@ -2267,6 +2382,11 @@ BOOST_AUTO_TEST_CASE( xml_chrono ) test_chrono(); } +BOOST_AUTO_TEST_CASE( json_chrono ) +{ + test_chrono(); +} + // ###################################################################### class SpecializedMSerialize { @@ -2389,6 +2509,11 @@ BOOST_AUTO_TEST_CASE( xml_structs_specialized ) test_structs_specialized(); } +BOOST_AUTO_TEST_CASE( json_structs_specialized ) +{ + test_structs_specialized(); +} + // ###################################################################### struct PolyBase { @@ -2498,3 +2623,8 @@ BOOST_AUTO_TEST_CASE( xml_polymorphic ) { test_polymorphic(); } + +BOOST_AUTO_TEST_CASE( json_polymorphic ) +{ + test_polymorphic(); +}