Remove rapidjson and fix time bug

Author: DraTeots

cpp/CMakeLists.txt

@@ -83,6 +83,7 @@ SET(TEST_SOURCE_FILES
         ${RCDB_HEADERS}
 
         tests/test_ConditionType.cpp
+        tests/test_Condition.cpp
 
         tests/catch.hpp
         tests/catch.cpp

cpp/examples/get_fadc_masks.cpp

@@ -40,6 +40,7 @@
 
 #include "RCDB/Connection.h"
 #include "RCDB/ConfigParser.h"
+#include <tao/json.hpp>
 
 
 using namespace std;
@@ -59,9 +60,9 @@ rcdb::ConfigFileParseResult GetMainConfigParsed(rcdb::Connection &connection, in
     }
 
 
-    auto json = rtvsCondition->ToJsonDocument();                        // The CODA rtvs is serialized as JSon dictionary.
+    auto json = tao::json::from_string(rtvsCondition->ToString());      // The CODA rtvs is serialized as JSon dictionary.
 
-    string fileName(json["%(config)"].GetString());                     // We need item with name '%(config)'
+    string fileName = json.at("%(config)").get_string();                // We need item with name '%(config)'
     // That is our file name
 
 
@@ -146,7 +147,7 @@ int main ( int argc, char *argv[] )
 
     // get all file names for this run
     cout<<"All files saved for run "<<runNumber<<endl;
-    for (int i = 0; i < runFileNames.size(); ++i) {
+    for (size_t i = 0; i < runFileNames.size(); ++i) {
         cout<<"    "<<runFileNames[i]<<endl;
     }
 
@@ -163,7 +164,7 @@ int main ( int argc, char *argv[] )
     auto comValues = comParseResult.SectionsBySlotNumber[3].NameVectors["FADC250_ALLCH_THR"];  // Parse it and return
 
     cout<<"FADC250_ALLCH_THR for slot 3 is:"<<endl;
-    for (int i = 0; i < comValues.size(); ++i) {
+    for (size_t i = 0; i < comValues.size(); ++i) {
         cout<<"    "<<comValues[i]<<endl;
     }
 
@@ -181,7 +182,7 @@ int main ( int argc, char *argv[] )
     auto userValues = userParseResult.SectionsBySlotNumber[3].NameVectors["FADC250_TRG_MASK"];  // Parse it and return
 
     cout<<"FADC250_TRG_MASK for slot 3 is:"<<endl;
-    for (int i = 0; i < userValues.size(); ++i) {
+    for (size_t i = 0; i < userValues.size(); ++i) {
         cout<<"    "<<userValues[i]<<endl;
     }
  // Lets say we want to get values for roc8

cpp/examples/get_trigger_params.cpp

@@ -38,6 +38,7 @@
 
 #include "RCDB/Connection.h"
 #include "RCDB/ConfigParser.h"
+#include <tao/json.hpp>
 
 using namespace std;
 
@@ -76,9 +77,9 @@ int main ( int argc, char *argv[] )
     }
 
 
-    auto json = rtvsCondition->ToJsonDocument();                        // The CODA rtvs is serialized as JSon dictionary.
+    auto json = tao::json::from_string(rtvsCondition->ToString());      // The CODA rtvs is serialized as JSon dictionary.
 
-    string fileName(json["%(config)"].GetString());                     // We need item with name '%(config)'
+    string fileName = json.at("%(config)").get_string();                // We need item with name '%(config)'
                                                                         // That is our file name
 
 

cpp/examples/write_array_to_json.cpp

@@ -1,22 +1,20 @@
 /**
- *  This is very simple example of how to save array to JSON using rapidjson
- *  Full RapidJson documentation is available here:
- *  http://rapidjson.org/index.html
+ *  This is very simple example of how to save an array to JSON.
+ *  RCDB bundles the taocpp/json library:
+ *  https://github.com/taocpp/json
  */
 #include <string>
 #include <iostream>
 
 #include "RCDB/WritingConnection.h"
 
-
-#include "rapidjson/stringbuffer.h"
-#include "rapidjson/writer.h"
+#include <tao/json.hpp>
 
 
 int main ( int argc, char *argv[] )
 {
     using namespace std;
-    using namespace rapidjson;
+    using namespace tao;
 
     // Get a connection string from arguments
     if ( argc != 2 ) {
@@ -60,19 +58,14 @@ int main ( int argc, char *argv[] )
     // P A R T   1 - w r i t i n g   a r r a y
 
     //We want to store some value and array to JSON file
-    Document document;
-    document.SetArray();                                // document must be SetArray or SetObject
-    auto& allocator = document.GetAllocator();          // You... just need this allocator. Imagine this is mantra
+    auto document = json::value::array({});             // a JSON value holding an (empty) array
     for(int i=-5; i<5; i++)
     {
-        document.PushBack(Value().SetInt(i), allocator);  // Put array values
+        document.emplace_back(i);                       // Put array values
     }
 
     // Convert document to string
-    StringBuffer buffer;
-    Writer<StringBuffer> writer(buffer);
-    document.Accept(writer);
-    string output = buffer.GetString();
+    string output = tao::json::to_string(document);
 
     // Print the JSon we've got
     cout<<"Resulting json is:"<<endl;
@@ -83,14 +76,12 @@ int main ( int argc, char *argv[] )
 
     // P A R T   2 - r e a d i n g   a r r a y
     auto cnd = connection.GetCondition(999, "json_cnd");
-    auto json = cnd->ToJsonDocument();
-
-    //string fileName(json["%(config)"].GetString());                     // We need item with name '%(config)'
+    auto json = tao::json::from_string(cnd->ToString());
 
     // since we saved json as array, we can iterate it directly
-    for(int i=0; i<json.Size(); i++)
+    for(const auto& item : json.get_array())
     {
-        std::cout<<" "<< json[i].GetInt();
+        std::cout<<" "<< item.get_signed();
     }
     std::cout<<endl;
 

cpp/examples/write_objects_to_json.cpp

@@ -14,7 +14,6 @@ using namespace tao;
 
 int main(int argc, char *argv[]) {
     using namespace std;
-    using namespace rapidjson;
 
     // Get a connection string from arguments
     if (argc != 2) {
@@ -144,7 +143,7 @@ id      name           value_type      created              description
     // explicit data conversion
     auto trigger = data["daq_trigger"].get_string();
 
-    cout<<"trigger= "<<trigger<<"   start_time="<<data["start_time"]<<endl;
+    cout<<"event_count= "<<ev_cnt<<"   trigger= "<<trigger<<"   start_time="<<data["start_time"]<<endl;
 
     return 0;
 }

cpp/include/RCDB/Condition.h

@@ -9,7 +9,7 @@
 #include "Exceptions.h"
 #include <chrono>
 #include <string>
-#include "rapidjson/document.h"
+#include <tao/json.hpp>   // ToJsonDocument() returns a tao::json::value (was rapidjson)
 
 class DataProvder;
 
@@ -74,32 +74,41 @@ namespace rcdb {
 
             if (GetValueType() != ValueTypes::Json &&
                 GetValueType() != ValueTypes::String &&
-                GetValueType() != ValueTypes::Blob) {
-                throw rcdb::ValueFormatError("Value type of the condition is not String, Json or Blob");
+                GetValueType() != ValueTypes::Blob &&
+                GetValueType() != ValueTypes::Time) {
+                throw rcdb::ValueFormatError("Value type of the condition is not String, Json, Blob or Time");
             }
 
+            // For Time values this is the raw stored datetime string
+            // ("YYYY-MM-DD HH:MM:SS"); use ToTime() for a parsed time_point.
             return _text_value;
         }
 
 
-        /// @deprecated use json ToJson() instead
-        rapidjson::Document ToJsonDocument()
+        /** Returns the condition value parsed as a JSON document.
+         *
+         *  Only valid when the condition's value type is Json (e.g. the CODA 'rtvs'
+         *  dictionary). This is a drop-in for the historical rapidjson implementation:
+         *  the same text is parsed into the same JSON structure/values -- only the
+         *  returned type changed from rapidjson::Document to tao::json::value, so the
+         *  header no longer depends on rapidjson. Callers keep `auto json = ...;` but
+         *  use tao's accessors (e.g. json.at("%(config)").get_string()) instead of
+         *  rapidjson's (json["%(config)"].GetString()).
+         *
+         *  Throws rcdb::ValueFormatError if the value type is not Json, or the text is
+         *  not parseable JSON -- the same error contract as the old implementation.
+         */
+        tao::json::value ToJsonDocument()
         {
-            using namespace rapidjson;
-
             if (GetValueType() != ValueTypes::Json) {
                 throw rcdb::ValueFormatError("Value type of the condition is not Json");
             }
 
-            Document document;  // Default template parameter uses UTF8 and MemoryPoolAllocator.
-
-
-            // "normal" parsing, decode strings to new buffers. Can use other input stream via ParseStream().
-            if (document.Parse(_text_value.c_str()).HasParseError()) {
+            try {
+                return tao::json::from_string(_text_value);
+            } catch (const std::exception &) {
                 throw rcdb::ValueFormatError("Error while parsing JSon");
             }
-
-            return document;
         }
 
 
@@ -123,36 +132,36 @@ namespace rcdb {
         rcdb::ValueTypes GetValueType() { return _type.GetValueType(); }
 
 
-        void SetId(unsigned long _id) {
-            Condition::_id = _id;
+        void SetId(unsigned long id) {
+            _id = id;
         }
 
         unsigned long GetId() {
             return _id;
         }
 
-        void SetRunNumber(unsigned long _runNumber) {
-            Condition::_runNumber = _runNumber;
+        void SetRunNumber(unsigned long runNumber) {
+            _runNumber = runNumber;
         }
 
-        void SetTextValue(const std::string &_text_value) {
-            Condition::_text_value = _text_value;
+        void SetTextValue(const std::string &text_value) {
+            _text_value = text_value;
         }
 
-        void SetIntValue(int _int_value) {
-            Condition::_int_value = _int_value;
+        void SetIntValue(int int_value) {
+            _int_value = int_value;
         }
 
-        void SetFloatValue(double _float_value) {
-            Condition::_float_value = _float_value;
+        void SetFloatValue(double float_value) {
+            _float_value = float_value;
         }
 
-        void SetBoolValue(bool _bool_value) {
-            Condition::_bool_value = _bool_value;
+        void SetBoolValue(bool bool_value) {
+            _bool_value = bool_value;
         }
 
-        void SetTime(std::chrono::time_point<std::chrono::system_clock> _time) {
-            Condition::_time = _time;
+        void SetTime(std::chrono::time_point<std::chrono::system_clock> time) {
+            _time = time;
         }
 
     private:

cpp/include/RCDB/ConfigParser.h

@@ -38,9 +38,9 @@ namespace rcdb
     {
     public:
 
-        ConfigFileParseResult(std::vector<std::string> SectionNames)
+        ConfigFileParseResult(std::vector<std::string> sectionNames)
         {
-            for(auto sectionName: SectionNames) {
+            for(auto sectionName: sectionNames) {
                 this->SectionNames.push_back(sectionName);
             }
         }

cpp/include/RCDB/MySqlProvider.h

@@ -136,11 +136,15 @@ namespace rcdb {
                         if (row[int_column] == nullptr) return std::unique_ptr<Condition>();
                         condition->SetIntValue(stoi(row[int_column]));
                         return condition;
-                    case ValueTypes::Time:
+                    case ValueTypes::Time: {
+                        // time is stored as a datetime string in time_value (col 5),
+                        // NOT as an integer in int_value -- read and parse the string.
                         if (row[time_column] == nullptr) return std::unique_ptr<Condition>();
-                        condition->SetTime(
-                                chrono::system_clock::from_time_t(stoul(row[int_column])));
+                        const std::string timeStr = row[time_column];
+                        condition->SetTextValue(timeStr);   // keep raw value for ToString()
+                        condition->SetTime(StringUtils::ParseTime(timeStr));
                         return condition;
+                    }
                     default:
                         throw std::logic_error("ValueTypes type is something different than one of possible values");
                 }

cpp/include/RCDB/SqLiteProvider.h

@@ -11,6 +11,7 @@
 #include <memory>
 #include "DataProvider.h"
 #include "RcdbFile.h"
+#include "StringUtils.h"
 
 namespace rcdb {
     class SqLiteProvider : public DataProvider {
@@ -112,12 +113,15 @@ namespace rcdb {
                         if(_getConditionQuery.isColumnNull(int_column)) return std::unique_ptr<Condition>();
                         condition->SetIntValue(_getConditionQuery.getColumn(int_column).getInt());
                         return condition;
-                    case ValueTypes::Time:
+                    case ValueTypes::Time: {
+                        // time is stored as a datetime string in time_value (col 5),
+                        // NOT as an integer in int_value -- read and parse the string.
                         if(_getConditionQuery.isColumnNull(time_column)) return std::unique_ptr<Condition>();
-                        condition->SetTime(
-                                std::chrono::system_clock::from_time_t(
-                                        _getConditionQuery.getColumn(int_column).getInt64()));
+                        const std::string timeStr = _getConditionQuery.getColumn(time_column).getText();
+                        condition->SetTextValue(timeStr);   // keep raw value for ToString()
+                        condition->SetTime(StringUtils::ParseTime(timeStr));
                         return condition;
+                    }
                     default:
                         throw std::logic_error("ValueTypes type is something different than one of possible values");
                 }
@@ -206,12 +210,15 @@ namespace rcdb {
                         if(_getConditionQuery.isColumnNull(int_column)) return std::unique_ptr<Condition>();
                         condition->SetIntValue(_getConditionQuery.getColumn(int_column).getInt());
                         return condition;
-                    case ValueTypes::Time:
+                    case ValueTypes::Time: {
+                        // time is stored as a datetime string in time_value (col 5),
+                        // NOT as an integer in int_value -- read and parse the string.
                         if(_getConditionQuery.isColumnNull(time_column)) return std::unique_ptr<Condition>();
-                        condition->SetTime(
-                                std::chrono::system_clock::from_time_t(
-                                        _getConditionQuery.getColumn(int_column).getInt64()));
+                        const std::string timeStr = _getConditionQuery.getColumn(time_column).getText();
+                        condition->SetTextValue(timeStr);   // keep raw value for ToString()
+                        condition->SetTime(StringUtils::ParseTime(timeStr));
                         return condition;
+                    }
                     default:
                         throw std::logic_error("ValueTypes type is something different than one of possible values");
                 }

cpp/include/RCDB/StringUtils.h

@@ -15,6 +15,11 @@
 #include <cctype>
 #include <locale>
 
+#include <chrono>
+#include <ctime>
+#include <iomanip>
+#include <sstream>
+
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -36,6 +41,28 @@ class StringUtils {
         return GetFormattedTime(*localtime(&time));
     }
 
+    /** Parses an RCDB datetime string into a system_clock time_point.
+     *
+     *  RCDB stores time-valued conditions (e.g. run_start_time) as a naive
+     *  wall-clock string "YYYY-MM-DD HH:MM:SS" (SQLite may append fractional
+     *  seconds, which are ignored). The naive value is interpreted as UTC so the
+     *  result is deterministic regardless of the reader's timezone --
+     *  system_clock::to_time_t() + gmtime() round-trips the stored fields. On an
+     *  unparseable string the epoch is returned.
+     */
+    static inline std::chrono::system_clock::time_point ParseTime(const std::string &formatted) {
+        std::tm tm = {};
+        std::istringstream ss(formatted);
+        ss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S");
+        if (ss.fail()) {
+            return std::chrono::system_clock::time_point{};   // epoch on bad input
+        }
+        // timegm() interprets the tm as UTC; available on Linux and macOS
+        // (RCDB C++ does not target Windows).
+        const std::time_t t = timegm(&tm);
+        return std::chrono::system_clock::from_time_t(t);
+    }
+
     // trim from start (in place)
     static inline void ltrim(std::string &s) {
         s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](char c) { return !isspace(c); }));