Support for mongodb maximum fetch feature (including parameter in order to define the maximum nb of records that can be fetch)

vertigo17 · vertigo17 · commit 6f65a965f742 · 2026-04-12T13:20:24.000+02:00
parameter cerberus_testdatalib_fetchmax renamed to cerberus_testdatalib_sql_fetchmax and cerberus_testdatalib_mongodb_fetchmax created.
diff --git a/source/src/main/java/org/cerberus/core/crud/entity/Parameter.java b/source/src/main/java/org/cerberus/core/crud/entity/Parameter.java
@@ -98,6 +98,9 @@ public class Parameter {
     public static final String VALUE_cerberus_gitlab_apitoken = "cerberus_gitlab_apitoken";
     public static final String VALUE_cerberus_use_w3c_capabilities = "cerberus_use_w3c_capabilities";
 
+    public static final String VALUE_cerberus_testdatalib_sql_fetchmax = "cerberus_testdatalib_sql_fetchmax";
+    public static final String VALUE_cerberus_testdatalib_mongodb_fetchmax = "cerberus_testdatalib_mongodb_fetchmax";
+    
     public static final String VALUE_cerberus_tagcombofilterpersystem_boolean = "cerberus_tagcombofilterpersystem_boolean";
 
     public static final String VALUE_cerberus_automatescore_changehorizon = "cerberus_automatescore_changehorizon";
diff --git a/source/src/main/java/org/cerberus/core/crud/service/impl/ParameterService.java b/source/src/main/java/org/cerberus/core/crud/service/impl/ParameterService.java
@@ -483,7 +483,8 @@ public boolean isSystemManaged(Parameter parameter) {
             case "cerberus_proxy_nonproxyhosts":
             case "cerberus_proxy_port":
             case "cerberus_propertyexternalsql_timeout":
-            case "cerberus_testdatalib_fetchmax":
+            case Parameter.VALUE_cerberus_testdatalib_sql_fetchmax:
+            case Parameter.VALUE_cerberus_testdatalib_mongodb_fetchmax:
             case "cerberus_notification_disableenvironment_body":
             case "cerberus_notification_disableenvironment_cc":
             case "cerberus_notification_disableenvironment_subject":
diff --git a/source/src/main/java/org/cerberus/core/crud/service/impl/TestDataLibService.java b/source/src/main/java/org/cerberus/core/crud/service/impl/TestDataLibService.java
@@ -24,6 +24,7 @@
 import org.apache.commons.fileupload.FileItem;
 import org.cerberus.core.crud.dao.ITestCaseCountryPropertiesDAO;
 import org.cerberus.core.crud.dao.ITestDataLibDAO;
+import org.cerberus.core.crud.entity.Parameter;
 import org.cerberus.core.crud.entity.TestCaseExecution;
 import org.cerberus.core.engine.entity.MessageEvent;
 import org.cerberus.core.engine.entity.MessageGeneral;
@@ -108,7 +109,7 @@ public AnswerList<HashMap<String, String>> readINTERNALWithSubdataByCriteria(Str
         List<HashMap<String, String>> result = new ArrayList<>();
 
         // We start by calculating the max nb of row we can fetch. Either specified by rowLimit either defined by a parameter.
-        int maxSecurityFetch = parameterService.getParameterIntegerByKey("cerberus_testdatalib_fetchmax", system, 100);
+        int maxSecurityFetch = parameterService.getParameterIntegerByKey(Parameter.VALUE_cerberus_testdatalib_sql_fetchmax, system, 100);
         int maxFetch;
         if (rowLimit > 0 && rowLimit < maxSecurityFetch) {
             maxFetch = rowLimit;
diff --git a/source/src/main/java/org/cerberus/core/database/DocumentationDatabaseService.java b/source/src/main/java/org/cerberus/core/database/DocumentationDatabaseService.java
diff --git a/source/src/main/java/org/cerberus/core/engine/gwt/impl/ActionService.java b/source/src/main/java/org/cerberus/core/engine/gwt/impl/ActionService.java
@@ -1859,7 +1859,7 @@ private MessageEvent doActionCallService(TestCaseStepActionExecution action, Str
         TestCaseExecution execution = action.getTestCaseStepExecution().gettCExecution();
         AnswerItem lastServiceCalledAnswer;
 
-        lastServiceCalledAnswer = serviceService.callService(value1, value2, value3, null, null, null, null, execution, robotServerService.getFromOptions(action.getOptions(), RobotServerService.OPTIONS_TIMEOUT_SYNTAX));
+        lastServiceCalledAnswer = serviceService.callService(value1, value2, value3, null, null, null, null, execution, robotServerService.getFromOptions(action.getOptions(), RobotServerService.OPTIONS_TIMEOUT_SYNTAX), 0);
         message = lastServiceCalledAnswer.getResultMessage();
 
         if (lastServiceCalledAnswer.getItem() != null) {
diff --git a/source/src/main/java/org/cerberus/core/service/appservice/IServiceService.java b/source/src/main/java/org/cerberus/core/service/appservice/IServiceService.java
@@ -47,9 +47,11 @@ public interface IServiceService {
      * @param manualOperation Used when service not defined.
      * @param execution
      * @param timeoutMs
+     * @param rowLimit
      * @return
      */
-    AnswerItem<AppService> callService(String service, String targetNbEvents, String targetNbSec, String database, String manualRequest, String manualServicePath, String manualOperation, TestCaseExecution execution, int timeoutMs);
+    AnswerItem<AppService> callService(String service, String targetNbEvents, String targetNbSec, String database, String manualRequest, String manualServicePath, String manualOperation, 
+            TestCaseExecution execution, int timeoutMs, int rowLimit);
 
     /**
      * Simulate a call from outside a normal testcase execution.
diff --git a/source/src/main/java/org/cerberus/core/service/appservice/impl/ServiceService.java b/source/src/main/java/org/cerberus/core/service/appservice/impl/ServiceService.java
@@ -114,7 +114,8 @@ public class ServiceService implements IServiceService {
     private ICountryEnvironmentDatabaseService countryEnvironmentDatabaseService;
 
     @Override
-    public AnswerItem<AppService> callService(String service, String targetNbEvents, String targetNbSec, String database, String manualRequest, String manualServicePathParam, String manualOperation, TestCaseExecution execution, int timeoutMs) {
+    public AnswerItem<AppService> callService(String service, String targetNbEvents, String targetNbSec, String database, String manualRequest, String manualServicePathParam, String manualOperation,
+            TestCaseExecution execution, int timeoutMs, int rowLimit) {
         MessageEvent message = new MessageEvent(MessageEventEnum.ACTION_FAILED_CALLSERVICE);
         String decodedRequest;
         String decodedRequestExtra1;
@@ -517,7 +518,7 @@ public AnswerItem<AppService> callService(String service, String targetNbEvents,
                                  * Call MONGODB and store it into the execution.
                                  */
                                 result = mongodbService.callMONGODB(decodedServicePath, decodedRequest, decodedRequestExtra1, appService.getMethod(),
-                                        appService.getOperation(), timeoutMs, system, execution);
+                                        appService.getOperation(), timeoutMs, system, execution, rowLimit);
                                 message = result.getResultMessage();
                                 message.resolveDescription("SERVICENAME", service);
                                 break;
@@ -892,7 +893,7 @@ public AnswerItem<AppService> callAPI(String service, String country, String env
         // We can now start the call with all data prepared.
         try {
 
-            ans = this.callService(service, kafkaNb, kafkaTime, null, "", "", "", execution, timeout);
+            ans = this.callService(service, kafkaNb, kafkaTime, null, "", "", "", execution, timeout, 0);
 
         } catch (Exception e) {
             LOG.error(e, e);
diff --git a/source/src/main/java/org/cerberus/core/service/datalib/impl/DataLibService.java b/source/src/main/java/org/cerberus/core/service/datalib/impl/DataLibService.java
@@ -855,7 +855,7 @@ private AnswerList<HashMap<String, String>> getDataObjectList(TestDataLib lib, i
                 columnsToHide = getListOfSecrets(lib.getTestDataLibID());
 
                 // Service Call is made here.
-                AnswerItem ai = serviceService.callService(lib.getService(), null, null, lib.getDatabaseUrl(), lib.getEnvelope(), lib.getServicePath(), lib.getMethod(), execution, 0);
+                AnswerItem ai = serviceService.callService(lib.getService(), null, null, lib.getDatabaseUrl(), lib.getEnvelope(), lib.getServicePath(), lib.getMethod(), execution, 0, rowLimit);
 
                 msg = ai.getResultMessage();
 
diff --git a/source/src/main/java/org/cerberus/core/service/mongodb/IMongodbService.java b/source/src/main/java/org/cerberus/core/service/mongodb/IMongodbService.java
@@ -40,8 +40,9 @@ public interface IMongodbService {
      * @param timeOutMs
      * @param system
      * @param tcexecution
+     * @param rowLimit
      * @return
      */
-    AnswerItem<AppService> callMONGODB(String servicePath, String queryString, String requestExtra1String, String method, String operation, int timeOutMs, String system, TestCaseExecution tcexecution);
+    AnswerItem<AppService> callMONGODB(String servicePath, String queryString, String requestExtra1String, String method, String operation, int timeOutMs, String system, TestCaseExecution tcexecution, int rowLimit);
 
 }
diff --git a/source/src/main/java/org/cerberus/core/service/mongodb/impl/MongodbService.java b/source/src/main/java/org/cerberus/core/service/mongodb/impl/MongodbService.java
@@ -33,7 +33,6 @@
 import com.mongodb.client.result.UpdateResult;
 import java.sql.Timestamp;
 import java.util.Date;
-import java.util.Iterator;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.cerberus.core.crud.entity.AppService;
@@ -52,6 +51,7 @@
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import org.bson.Document;
+import org.cerberus.core.crud.entity.Parameter;
 import org.cerberus.core.service.mongodb.IMongodbService;
 import org.cerberus.core.util.StringUtil;
 import org.json.JSONArray;
@@ -77,10 +77,11 @@ public class MongodbService implements IMongodbService {
     IProxyService proxyService;
 
     private static final Logger LOG = LogManager.getLogger(MongodbService.class);
+    private static final int MAW_ROW = 10000;
 
     @Override
     public AnswerItem<AppService> callMONGODB(String servicePath, String requestString, String requestExtra1String, String method, String operation, int timeOutMs,
-            String system, TestCaseExecution tcexecution) {
+            String system, TestCaseExecution tcexecution, int rowLimit) {
         MessageEvent message = null;
         AnswerItem<AppService> result = new AnswerItem<>();
 
@@ -165,23 +166,31 @@ public AnswerItem<AppService> callMONGODB(String servicePath, String requestStri
             switch (method) {
 
                 case AppService.METHOD_MONGODBFIND:
-                    try (
-                            MongoCursor<Document> cursor = collection.find(BasicDBObject.parse(requestString))
-                                    .iterator()) {
-                                int i = 0;
-                                while (cursor.hasNext() && i < 5) {
-                                    LOG.debug("Results found.");
-                                    mongoDBResult = cursor.next().toJson();
-                                    i++;
-                                    mongoDBResultArray.put(new JSONObject(mongoDBResult));
-                                    LOG.debug(mongoDBResult);
+
+                    int maxSecurityFetch = parameterService.getParameterIntegerByKey(Parameter.VALUE_cerberus_testdatalib_mongodb_fetchmax, system, 10000);
+                    int maxFetch;
+                    if (rowLimit > 0 && rowLimit < maxSecurityFetch) {
+                        maxFetch = rowLimit;
+                    } else {
+                        maxFetch = maxSecurityFetch;
+                    }
+
+                    try (MongoCursor<Document> cursor = collection.find(BasicDBObject.parse(requestString)).limit(maxFetch)
+                            .iterator()) {
+                        int i = 0;
+                        while (cursor.hasNext() && i < 5) {
+                            LOG.debug("Results found.");
+                            mongoDBResult = cursor.next().toJson();
+                            i++;
+                            mongoDBResultArray.put(new JSONObject(mongoDBResult));
+                            LOG.debug(mongoDBResult);
 //                    System.out.println(cursor.next().toJson());
-                                }
-                                serviceMONGODB.setResponseHTTPBody(mongoDBResultArray.toString());
-                                serviceMONGODB.setResponseNb(i);
-                                serviceMONGODB.setEnd(new Timestamp(new Date().getTime()));
+                        }
+                        serviceMONGODB.setResponseHTTPBody(mongoDBResultArray.toString());
+                        serviceMONGODB.setResponseNb(i);
+                        serviceMONGODB.setEnd(new Timestamp(new Date().getTime()));
 
-                            }
+                    }
                     break;
 
                 case AppService.METHOD_MONGODBUPDATEONE:
@@ -214,15 +223,15 @@ public AnswerItem<AppService> callMONGODB(String servicePath, String requestStri
             result.setResultMessage(message);
             serviceMONGODB.setEnd(new Timestamp(new Date().getTime()));
             return result;
-            
+
         } catch (MongoSocketOpenException ex) {
             LOG.info("Exception when performing the MONGODB Call. " + ex.toString());
             message = new MessageEvent(MessageEventEnum.ACTION_FAILED_CALLSERVICE);
             message.resolveDescription("DESCRIPTION", ex.toString());
             result.setResultMessage(message);
             serviceMONGODB.setEnd(new Timestamp(new Date().getTime()));
             return result;
-            
+
         } catch (Exception ex) {
             LOG.error(ex, ex);
             LOG.info("Exception when performing the MONGODB Call. " + ex.toString());
diff --git a/source/src/main/java/org/cerberus/core/service/sql/impl/SQLService.java b/source/src/main/java/org/cerberus/core/service/sql/impl/SQLService.java
@@ -34,6 +34,7 @@
 import org.apache.logging.log4j.Logger;
 import org.cerberus.core.config.cerberus.Property;
 import org.cerberus.core.crud.entity.CountryEnvironmentDatabase;
+import org.cerberus.core.crud.entity.Parameter;
 import org.cerberus.core.engine.entity.MessageEvent;
 import org.cerberus.core.crud.entity.TestCaseCountryProperties;
 import org.cerberus.core.crud.entity.TestCaseExecution;
@@ -411,7 +412,7 @@ public MessageEvent executeCallableStatement(String system, String country, Stri
     public AnswerList<HashMap<String, String>> queryDatabaseNColumns(String connectionName, String sql, int rowLimit, int defaultTimeOut, String system, HashMap<String, String> columnsToGet, List<String> columnsToHide, boolean ignoreNoMatchColumns, String defaultNoMatchColumnValue, TestCaseExecution execution) {
         AnswerList<HashMap<String, String>> listResult = new AnswerList<>();
         List<HashMap<String, String>> list;
-        int maxSecurityFetch = parameterService.getParameterIntegerByKey("cerberus_testdatalib_fetchmax", system, 100);
+        int maxSecurityFetch = parameterService.getParameterIntegerByKey(Parameter.VALUE_cerberus_testdatalib_sql_fetchmax, system, 100);
         int maxFetch;
         if (rowLimit > 0 && rowLimit < maxSecurityFetch) {
             maxFetch = rowLimit;
diff --git a/source/src/main/resources/database.sql b/source/src/main/resources/database.sql
@@ -6816,3 +6816,10 @@ INSERT INTO `invariant` (`idname`, `value`, `sort`, `description`, `VeryShortDes
 -- 1935
 ALTER TABLE `robot`
     ADD COLUMN `preloadScript` TEXT NULL DEFAULT NULL AFTER `IsAcceptInsecureCerts`;
+
+-- 1936
+INSERT INTO `parameter` (`system`, `param`, `value`, `description`)
+  VALUES ('', 'cerberus_testdatalib_mongodb_fetchmax', '10000', 'Maximum number of fetched records that Cerberus will perform when retrieving a data from a Mongodg service Data Library.');
+
+-- 1937
+UPDATE `parameter` SET `param`='cerberus_testdatalib_sql_fetchmax' WHERE `param`='cerberus_testdatalib_fetchmax';
