Added family map option to not use chunk size for distribution.

Author: jgates108

src/cconfig/CzarConfig.h

@@ -221,6 +221,9 @@ class CzarConfig {
     /// Return the sleep time (in milliseconds) between messages sent to active workers.
     int getMonitorSleepTimeMilliSec() const { return _monitorSleepTimeMilliSec->getVal(); }
 
+    /// Return true if family map chunk distribution should depend on chunk size.
+    bool getFamilyMapUsingChunkSize() const { return _familyMapUsingChunkSize->getVal(); }
+
     // Parameters of the Czar management service
 
     std::string const& replicationInstanceId() const { return _replicationInstanceId->getVal(); }
@@ -424,6 +427,10 @@ class CzarConfig {
     CVTIntPtr _monitorSleepTimeMilliSec = util::ConfigValTInt::create(
             _configValMap, "activeworker", "monitorSleepTimeMilliSec", notReq, 15'000);
 
+    // FamilyMap
+    CVTBoolPtr _familyMapUsingChunkSize =
+            util::ConfigValTBool::create(_configValMap, "familymap", "usingChunkSize", notReq, 0);
+
     /// This may impact `_resultMaxHttpConnections` as too many connections may cause kernel memory issues.
     CVTIntPtr _commandMaxHttpConnections =
             util::ConfigValTInt::create(_configValMap, "uberjob", "commandMaxHttpConnections", notReq, 2000);

src/czar/CzarChunkMap.cc

@@ -31,6 +31,7 @@
 
 // Qserv headers
 #include "qmeta/QMeta.h"
+#include "cconfig/CzarConfig.h"
 #include "czar/Czar.h"
 #include "czar/CzarRegistry.h"
 #include "qmeta/Exceptions.h"
@@ -342,7 +343,9 @@ bool CzarFamilyMap::_read() {
     }
 
     // Make the new maps.
-    shared_ptr<CzarFamilyMap::FamilyMapType> familyMapPtr = makeNewMaps(qChunkMap);
+    auto czConfig = cconfig::CzarConfig::instance();
+    bool usingChunkSize = czConfig->getFamilyMapUsingChunkSize();
+    shared_ptr<CzarFamilyMap::FamilyMapType> familyMapPtr = makeNewMaps(qChunkMap, usingChunkSize);
 
     verify(familyMapPtr);
 
@@ -355,7 +358,7 @@ bool CzarFamilyMap::_read() {
 }
 
 std::shared_ptr<CzarFamilyMap::FamilyMapType> CzarFamilyMap::makeNewMaps(
-        qmeta::QMetaChunkMap const& qChunkMap) {
+        qmeta::QMetaChunkMap const& qChunkMap, bool usingChunkSize) {
     // Create new maps.
     std::shared_ptr<FamilyMapType> newFamilyMap = make_shared<FamilyMapType>();
 
@@ -370,7 +373,10 @@ std::shared_ptr<CzarFamilyMap::FamilyMapType> CzarFamilyMap::makeNewMaps(
                 for (qmeta::QMetaChunkMap::ChunkInfo const& chunkInfo : chunks) {
                     try {
                         int64_t chunkNum = chunkInfo.chunk;
-                        CzarChunkMap::SizeT sz = chunkInfo.size;
+                        CzarChunkMap::SizeT sz = 1;
+                        if (usingChunkSize) {
+                            sz = chunkInfo.size;
+                        }
                         LOGS(_log, LOG_LVL_DEBUG,
                              cName(__func__) << "workerdId=" << workerId << " db=" << dbName << " table="
                                              << tableName << " chunk=" << chunkNum << " sz=" << sz);

src/czar/CzarChunkMap.h

@@ -298,10 +298,13 @@ class CzarFamilyMap {
 
     /// Make a new FamilyMapType map including ChunkMap and WorkerChunkMap from the data
     /// in `qChunkMap`. Each family has its own ChunkMap and WorkerChunkMap.
+    /// @param qChunkMap - data source for the family map
+    /// @param usingChunkSize - true if the distribution of chunks will depend on the
+    ///                  size of the chunks/
     ///
     /// NOTE: This is likely an expensive operation and should probably only
     ///   be called if new workers have been added or chunks have been moved.
-    std::shared_ptr<FamilyMapType> makeNewMaps(qmeta::QMetaChunkMap const& qChunkMap);
+    std::shared_ptr<FamilyMapType> makeNewMaps(qmeta::QMetaChunkMap const& qChunkMap, bool usingChunkSize);
 
     /// Insert the new element described by the parameters into the `newFamilyMap` as appropriate.
     void insertIntoMaps(std::shared_ptr<FamilyMapType> const& newFamilyMap, std::string const& workerId,

src/czar/testCzar.cc

@@ -191,13 +191,13 @@ BOOST_AUTO_TEST_CASE(CzarChunkMap) {
 
     auto jsTest1 = nlohmann::json::parse(test1);
     qmeta::QMetaChunkMap qChunkMap1 = convertJsonToChunkMap(jsTest1);
-    auto familyMap = czFamMap.makeNewMaps(qChunkMap1);
+    auto familyMap = czFamMap.makeNewMaps(qChunkMap1, true);
     czar::CzarFamilyMap::verify(familyMap);  // Throws on failure.
     LOGS(_log, LOG_LVL_DEBUG, "CzarFamilyMap test 1 passed");
 
     auto jsTest2 = nlohmann::json::parse(test2);
     qmeta::QMetaChunkMap qChunkMap2 = convertJsonToChunkMap(jsTest2);
-    auto familyMap2 = czFamMap.makeNewMaps(qChunkMap2);
+    auto familyMap2 = czFamMap.makeNewMaps(qChunkMap2, true);
     czar::CzarFamilyMap::verify(familyMap2);  // Throws on failure.
     LOGS(_log, LOG_LVL_DEBUG, "CzarFamilyMap test 2 passed");
 }