Skip to content

Commit 68be5c8

Browse files
eng-dedaluseng-dedalus
committed
[KNOWAGE-9160] add useGroupBy parameter to dataset strategies and update related methods
1 parent c21c8fe commit 68be5c8

17 files changed

Lines changed: 40 additions & 34 deletions

‎knowage-core/src/main/java/it/eng/knowage/backendservices/rest/widgets/PythonProxy.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ private Response executeWidget(String outputType, PythonWidgetDTO pythonWidgetDT
8888
Map<String, Object> drivers = pythonWidgetDTO.getDrivers();
8989
if (dsLabel != null) {
9090
datastore = getDataStore(dsLabel, pythonWidgetDTO.getParameters(), drivers, pythonWidgetDTO.getSelections(), null, -1,
91-
pythonWidgetDTO.getAggregations(), null, -1, -1, null, null);
91+
pythonWidgetDTO.getAggregations(), null, -1, -1, null, null, false);
9292
}
9393
body = PythonUtils.createPythonEngineRequestBody(datastore, dsLabel, pythonWidgetDTO.getScript(), drivers, pythonWidgetDTO.getOutputVariable());
9494
pythonAddress = PythonUtils.getPythonAddress(pythonWidgetDTO.getEnvironmentLabel());

‎knowage-core/src/main/java/it/eng/knowage/backendservices/rest/widgets/RWidgetProxy.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ public Response view(@PathParam("output_type") String outputType, Map<String, St
9090
}
9191
String rDataframe = null;
9292
if (dsLabel != null) {
93-
String knowageDs = getDataStore(dsLabel, parameters, drivers, selections, null, -1, aggregations, null, -1, -1, false, null, null);
93+
String knowageDs = getDataStore(dsLabel, parameters, drivers, selections, null, -1, aggregations, null, -1, -1, false, null, null, false);
9494
rDataframe = MLEngineUtils.dataStore2DataFrame(knowageDs);
9595
}
9696
it.eng.spagobi.utilities.rest.RestUtilities.Response rEngineResponse = null;
@@ -152,7 +152,7 @@ public Response edit(@PathParam("output_type") String outputType, Map<String, St
152152
}
153153
String rDataframe = null;
154154
if (dsLabel != null) {
155-
String knowageDs = getDataStore(dsLabel, parameters, drivers, selections, null, -1, aggregations, null, -1, -1, false, null, null);
155+
String knowageDs = getDataStore(dsLabel, parameters, drivers, selections, null, -1, aggregations, null, -1, -1, false, null, null, false);
156156
rDataframe = MLEngineUtils.dataStore2DataFrame(knowageDs);
157157
}
158158
it.eng.spagobi.utilities.rest.RestUtilities.Response rEngineResponse = null;

‎knowage-core/src/main/java/it/eng/knowage/document/export/cockpit/CSVCockpitDataExporter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ public UserProfile getUserProfile() {
128128
*/
129129
private IDataStore getDataStore(IDataStoreConfiguration conf) throws JSONException {
130130
return new DatasetManagementAPI(userProfile).getDataStore(conf.getDataset(), true, conf.getParameters(), conf.getProjections(), conf.getFilter(),
131-
conf.getGroups(), conf.getSortings(), conf.getSummaryRowProjections(), conf.getOffset(), conf.getFetchSize(), conf.getMaxRowCount(), null);
131+
conf.getGroups(), conf.getSortings(), conf.getSummaryRowProjections(), conf.getOffset(), conf.getFetchSize(), conf.getMaxRowCount(), null, false);
132132
}
133133

134134
/**

‎knowage-core/src/main/java/it/eng/spagobi/api/common/AbstractDataSetResource.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -174,21 +174,21 @@ protected Integer[] getIdsAsIntegers(String ids) {
174174
@Deprecated
175175
public String getDataStore(String label, String parameters, Map<String, Object> drivers, String selections,
176176
String likeSelections, int maxRowCount, String aggregations, String summaryRow, int offset, int fetchSize,
177-
Boolean isNearRealtime, Set<String> indexes, String widgetName) {
177+
Boolean isNearRealtime, Set<String> indexes, String widgetName, boolean useGroupBy) {
178178
return getDataStore(label, parameters, drivers, selections, likeSelections, maxRowCount, aggregations,
179-
summaryRow, offset, fetchSize, isNearRealtime, null, indexes, widgetName);
179+
summaryRow, offset, fetchSize, isNearRealtime, null, indexes, widgetName, useGroupBy);
180180
}
181181

182182
public String getDataStore(String label, String parameters, Map<String, Object> drivers, String selections,
183183
String likeSelections, int maxRowCount, String aggregations, String summaryRow, int offset, int fetchSize,
184-
Set<String> indexes, String widgetName) {
184+
Set<String> indexes, String widgetName, boolean useGroupBy) {
185185
return getDataStore(label, parameters, drivers, selections, likeSelections, maxRowCount, aggregations,
186-
summaryRow, offset, fetchSize, null, null, indexes, widgetName);
186+
summaryRow, offset, fetchSize, null, null, indexes, widgetName, useGroupBy);
187187
}
188188

189189
public String getDataStore(String label, String parameters, Map<String, Object> drivers, String selections,
190190
String likeSelections, int maxRowCount, String aggregations, String summaryRow, int offset, int fetchSize,
191-
Boolean isNearRealtime, String options, Set<String> indexes, String widgetName) {
191+
Boolean isNearRealtime, String options, Set<String> indexes, String widgetName, boolean useGroupBy) {
192192
LOGGER.debug("IN");
193193
DatasetManagementAPI datasetManagementAPI = getDatasetManagementAPI();
194194
Monitor totalTiming = MonitorFactory.start("Knowage.AbstractDataSetResource.getDataStore");
@@ -305,7 +305,7 @@ public String getDataStore(String label, String parameters, Map<String, Object>
305305
columnAliasToName);
306306

307307
IDataStore dataStore = datasetManagementAPI.getDataStore(dataSet, isNearRealtime, parametersMap,
308-
projections, where, groups, sortings, summaryRowArray, offset, fetchSize, maxRowCount, indexes);
308+
projections, where, groups, sortings, summaryRowArray, offset, fetchSize, maxRowCount, indexes, useGroupBy);
309309

310310
// if required apply function from catalog
311311
String catalogFuncId = getCatalogFunctionUuid(projections);
@@ -443,7 +443,7 @@ public String getDataStoreAI(String label, String parameters, Map<String, Object
443443
columnAliasToName);
444444

445445
IDataStore dataStore = datasetManagementAPI.getDataStore(dataSet, isNearRealtime, parametersMap,
446-
projections, where, groups, sortings, summaryRowArray, offset, fetchSize, maxRowCount, indexes);
446+
projections, where, groups, sortings, summaryRowArray, offset, fetchSize, maxRowCount, indexes, false);
447447

448448
// if required apply function from catalog
449449
String catalogFuncId = getCatalogFunctionUuid(projections);

‎knowage-core/src/main/java/it/eng/spagobi/api/v2/DataSetResource.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -712,7 +712,7 @@ public View openPreview() {
712712
public String getDataStorePostWithJsonInBody(@PathParam("label") String label, String body,
713713
@DefaultValue("-1") @QueryParam("limit") int maxRowCount,
714714
@DefaultValue("-1") @QueryParam("offset") int offset, @DefaultValue("-1") @QueryParam("size") int fetchSize,
715-
@QueryParam("nearRealtime") boolean isNearRealtime, @QueryParam("widgetName") String widgetName) {
715+
@QueryParam("nearRealtime") boolean isNearRealtime, @DefaultValue("false") @QueryParam("useGroupBy") boolean useGroupBy, @QueryParam("widgetName") String widgetName) {
716716
try {
717717
Monitor timing = MonitorFactory.start("Knowage.DataSetResource.getDataStorePostWithJsonInBody:parseInputs");
718718

@@ -774,7 +774,7 @@ public String getDataStorePostWithJsonInBody(@PathParam("label") String label, S
774774
}
775775
timing.stop();
776776
return getDataStore(label, parameters, driversRuntimeMap, selections, likeSelections, maxRowCount,
777-
aggregations, summaryRow, offset, fetchSize, isNearRealtime, options, columns, widgetName);
777+
aggregations, summaryRow, offset, fetchSize, isNearRealtime, options, columns, widgetName, useGroupBy);
778778
} catch (CatalogFunctionException e) {
779779
throw e;
780780
} catch (Exception e) {
@@ -887,7 +887,7 @@ public String getDataStorePreview(@PathParam("label") String label, String body)
887887

888888
timing.stop();
889889
return getDataStore(label, parameters, driversRuntimeMap, null, likeSelections, -1, aggregations, null,
890-
start, limit, columns, null);
890+
start, limit, columns, null, false);
891891
} catch (JSONException e) {
892892
throw new SpagoBIRestServiceException(buildLocaleFromSession(), e);
893893
} catch (Exception e) {

‎knowage-core/src/main/java/it/eng/spagobi/tools/dataset/DatasetManagementAPI.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ public String persistDataset(String label) throws Exception {
289289
public IDataStore getDataStore(IDataSet dataSet, boolean isNearRealtime, Map<String, String> parametersValues,
290290
List<AbstractSelectionField> projections, Filter filter, List<AbstractSelectionField> list,
291291
List<Sorting> sortings, List<List<AbstractSelectionField>> summaryRowProjections, int offset, int fetchSize,
292-
int maxRowCount, Set<String> indexes) throws JSONException {
292+
int maxRowCount, Set<String> indexes, boolean useGroupBy) throws JSONException {
293293

294294
Monitor totalTiming = MonitorFactory.start("Knowage.DatasetManagementAPI.getDataStore");
295295
try {
@@ -299,7 +299,7 @@ public IDataStore getDataStore(IDataSet dataSet, boolean isNearRealtime, Map<Str
299299
IDatasetEvaluationStrategy strategy = DatasetEvaluationStrategyFactory
300300
.get(dataSet.getEvaluationStrategy(isNearRealtime), dataSet, userProfile);
301301
return strategy.executeQuery(projections, filter, list, sortings, summaryRowProjections, offset, fetchSize,
302-
maxRowCount, indexes);
302+
maxRowCount, indexes, useGroupBy);
303303

304304
} finally {
305305
totalTiming.stop();
@@ -910,7 +910,7 @@ public List<Filter> calculateMinMaxFilters(IDataSet dataSet, boolean isNearRealt
910910
IDataStore dataStore = null;
911911
if (dataSet.getEvaluationStrategy(isNearRealtime).equals(DatasetEvaluationStrategyType.CACHED)) {
912912
dataStore = getDataStore(dataSet, isNearRealtime, parametersValues, minMaxProjections, where, null,
913-
null, null, -1, -1, -1, indexes);
913+
null, null, -1, -1, -1, indexes, false);
914914
} else {
915915
// List<List<Projection>> listToPrj = new ArrayList<List<Projection>>();
916916
// listToPrj.add(minMaxProjections);

‎knowage-core/src/main/java/it/eng/spagobi/tools/dataset/cache/impl/sqldbcache/SQLDBCache.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,7 @@ private IDataStore queryStandardCachedDataset(IDataSet dataSet, String resultset
323323

324324
IDatasetEvaluationStrategy strategy = DatasetEvaluationStrategyFactory.get(DatasetEvaluationStrategyType.FLAT, flatDataSet, null);
325325

326-
toReturn = strategy.executeQuery(projections, filter, groups, sortings, summaryRowProjections, offset, fetchSize, maxRowCount, indexes);
326+
toReturn = strategy.executeQuery(projections, filter, groups, sortings, summaryRowProjections, offset, fetchSize, maxRowCount, indexes, false);
327327
toReturn.setCacheDate(cacheItem.getCreationDate());
328328

329329
/* CHECK IF INDEXES EXIST OR CREATE THEM */

‎knowage-core/src/main/java/it/eng/spagobi/tools/dataset/strategy/AbstractEvaluationStrategy.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ protected AbstractEvaluationStrategy(IDataSet dataSet) {
6161

6262
@Override
6363
public IDataStore executeQuery(List<AbstractSelectionField> projections, Filter filter, List<AbstractSelectionField> groups, List<Sorting> sortings,
64-
List<List<AbstractSelectionField>> summaryRowProjections, int offset, int fetchSize, int maxRowCount, Set<String> indexes) {
64+
List<List<AbstractSelectionField>> summaryRowProjections, int offset, int fetchSize, int maxRowCount, Set<String> indexes, boolean useGroupBy) {
6565
IDataStore dataStore;
6666
if (isUnsatisfiedFilter(filter)) {
6767
/*
@@ -89,7 +89,7 @@ public IDataStore executeQuery(List<AbstractSelectionField> projections, Filter
8989
dataStore = new DataStore(newMeta);
9090
} else {
9191
List<AbstractSelectionField> newProjections = applyTotalsFunctionsToFormulas(dataSet, projections, filter, maxRowCount, indexes);
92-
dataStore = execute(newProjections, filter, groups, sortings, summaryRowProjections, offset, fetchSize, maxRowCount, indexes);
92+
dataStore = execute(newProjections, filter, groups, sortings, summaryRowProjections, offset, fetchSize, maxRowCount, indexes, useGroupBy);
9393
IMetaData dataStoreToUseMeta = dataStore.getMetaData();
9494
if (!isSummaryRowIncluded() && summaryRowProjections != null && !summaryRowProjections.isEmpty()) {
9595
int i = 0;
@@ -211,7 +211,7 @@ public IDataStore executeSummaryRowQuery(List<AbstractSelectionField> summaryRow
211211
}
212212

213213
protected abstract IDataStore execute(List<AbstractSelectionField> projections, Filter filter, List<AbstractSelectionField> groups, List<Sorting> sortings,
214-
List<List<AbstractSelectionField>> summaryRowProjections, int offset, int fetchSize, int maxRowCount, Set<String> indexes);
214+
List<List<AbstractSelectionField>> summaryRowProjections, int offset, int fetchSize, int maxRowCount, Set<String> indexes, boolean useGroupBy);
215215

216216
protected abstract IDataStore executeSummaryRow(List<AbstractSelectionField> summaryRowProjections, IMetaData metaData, Filter filter, int maxRowCount);
217217

‎knowage-core/src/main/java/it/eng/spagobi/tools/dataset/strategy/AbstractJdbcEvaluationStrategy.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,16 +44,22 @@ protected AbstractJdbcEvaluationStrategy(IDataSet dataSet) {
4444

4545
@Override
4646
protected IDataStore execute(List<AbstractSelectionField> projections, Filter filter, List<AbstractSelectionField> groups, List<Sorting> sortings,
47-
List<List<AbstractSelectionField>> summaryRowProjections, int offset, int fetchSize, int maxRowCount, Set<String> indexes) {
47+
List<List<AbstractSelectionField>> summaryRowProjections, int offset, int fetchSize, int maxRowCount, Set<String> indexes, boolean useGroupBy) {
4848
LOGGER.debug("IN");
4949

5050
IDataStore pagedDataStore;
5151

5252
try {
53-
List<AbstractSelectionField> effectiveGroups = (groups != null && !groups.isEmpty()) ? groups : projections;
53+
SelectQuery selectQuery;
54+
if (useGroupBy) {
55+
List<AbstractSelectionField> effectiveGroups = (groups != null && !groups.isEmpty()) ? groups : projections;
56+
selectQuery = new SelectQuery(dataSet).select(projections).from(getTableName()).where(filter).groupBy(effectiveGroups)
57+
.orderBy(sortings);
58+
} else {
59+
selectQuery = new SelectQuery(dataSet).selectDistinct().select(projections).from(getTableName()).where(filter).groupBy(groups)
60+
.orderBy(sortings);
61+
}
5462

55-
SelectQuery selectQuery = new SelectQuery(dataSet).select(projections).from(getTableName()).where(filter).groupBy(effectiveGroups)
56-
.orderBy(sortings);
5763
pagedDataStore = getDataSource().executeStatement(selectQuery, offset, fetchSize, maxRowCount, true);
5864
pagedDataStore.setCacheDate(getDate());
5965
} catch (DataBaseException e) {

‎knowage-core/src/main/java/it/eng/spagobi/tools/dataset/strategy/CachedEvaluationStrategy.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public CachedEvaluationStrategy(UserProfile userProfile, IDataSet dataSet, ICach
6060

6161
@Override
6262
protected IDataStore execute(List<AbstractSelectionField> projections, Filter filter, List<AbstractSelectionField> groups, List<Sorting> sortings,
63-
List<List<AbstractSelectionField>> summaryRowProjections, int offset, int fetchSize, int maxRowCount, Set<String> indexes) {
63+
List<List<AbstractSelectionField>> summaryRowProjections, int offset, int fetchSize, int maxRowCount, Set<String> indexes, boolean useGroupBy) {
6464
Monitor totalCacheTiming = MonitorFactory.start("Knowage.DatasetManagementAPI.getDataStore:totalCache");
6565
IDataStore dataStore;
6666
try {

0 commit comments

Comments
 (0)