Skip to content

Commit aab47d1

Browse files
bharath-techiebharath-techie
committed
Some perf improvement fixes
Signed-off-by: Bharathwaj G <bharath78910@gmail.com>
1 parent 197d659 commit aab47d1

5 files changed

Lines changed: 54 additions & 37 deletions

File tree

‎server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ public abstract class BaseSingleTreeBuilder {
155155
}
156156

157157
// TODO : Removing hardcoding
158-
_maxLeafRecords = 1000; // builderConfig.getMaxLeafRecords();
158+
_maxLeafRecords = 100; // builderConfig.getMaxLeafRecords();
159159
}
160160

161161
private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException {

‎server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java‎

Lines changed: 26 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -96,10 +96,7 @@ public StarTreeFilter(
9696

9797
// 1706268600 / (60*60*1000) * (60*60*1000)
9898
public DocIdSetIterator getStarTreeResult() throws IOException {
99-
long startTime = System.nanoTime();
10099
StarTreeResult starTreeResult = traverseStarTree();
101-
logger.info("Star tree traversal took : {}", System.nanoTime() - startTime);
102-
startTime = System.nanoTime();
103100
List<DocIdSetIterator> andIterators = new ArrayList<>();
104101
andIterators.add(starTreeResult._matchedDocIds.build().iterator());
105102
DocIdSetIterator docIdSetIterator = andIterators.get(0);
@@ -109,31 +106,33 @@ public DocIdSetIterator getStarTreeResult() throws IOException {
109106
DocIdSetBuilder builder = new DocIdSetBuilder(starTreeResult.numOfMatchedDocs);
110107
List<Predicate<Long>> compositePredicateEvaluators = _predicateEvaluators.get(remainingPredicateColumn);
111108
SortedNumericDocValues ndv = this.dimValueMap.get(remainingPredicateColumn);
112-
long ndvStartTime1 = System.nanoTime();
109+
List<Integer> docIds = new ArrayList<>();
113110
while (docIdSetIterator.nextDoc() != NO_MORE_DOCS) {
114111
docCount++;
115-
long ndvStartTime = System.nanoTime();
116112
int docID = docIdSetIterator.docID();
117-
ndv.advanceExact(docID);
118-
long value = ndv.nextValue();
119-
logger.info("Advancing took : {}", System.nanoTime() - ndvStartTime);
120-
ndvStartTime = System.nanoTime();
121-
for (Predicate<Long> compositePredicateEvaluator : compositePredicateEvaluators) {
122-
// TODO : this might be expensive as its done against all doc values docs
123-
if (compositePredicateEvaluator.test(value)) {
124-
builder.grow(1).add(docID);
125-
break;
113+
if(ndv.advanceExact(docID)) {
114+
final int valuesCount = ndv.docValueCount();
115+
long value = ndv.nextValue();
116+
for (Predicate<Long> compositePredicateEvaluator : compositePredicateEvaluators) {
117+
// TODO : this might be expensive as its done against all doc values docs
118+
if (compositePredicateEvaluator.test(value)) {
119+
docIds.add(docID);
120+
for (int i = 0; i < valuesCount - 1; i++) {
121+
while(docIdSetIterator.nextDoc() != NO_MORE_DOCS) {
122+
docIds.add(docIdSetIterator.docID());
123+
}
124+
}
125+
break;
126+
}
126127
}
127128
}
128-
logger.info("Predicate took : {}", System.nanoTime() - ndvStartTime);
129129
}
130-
logger.info("Overall ndv took : {}", System.nanoTime() - ndvStartTime1);
131-
long buildTime = System.nanoTime();
130+
DocIdSetBuilder.BulkAdder adder = builder.grow(docIds.size());
131+
for(int docID : docIds) {
132+
adder.add(docID);
133+
}
132134
docIdSetIterator = builder.build().iterator();
133-
logger.info("Builder took : {}", System.nanoTime() - buildTime);
134135
}
135-
logger.info("Doc value num : {}" , docCount);
136-
logger.info("Rest of tree traversal took : {}", System.nanoTime() - startTime);
137136
return docIdSetIterator;
138137
}
139138

@@ -168,6 +167,7 @@ private StarTreeResult traverseStarTree() throws IOException {
168167
}
169168

170169
StarTreeNode starTreeNode;
170+
List<Integer> docIds = new ArrayList<>();
171171
while ((starTreeNode = queue.poll()) != null) {
172172
int dimensionId = starTreeNode.getDimensionId();
173173
if (dimensionId > currentDimensionId) {
@@ -183,9 +183,8 @@ private StarTreeResult traverseStarTree() throws IOException {
183183

184184
// If all predicate columns and group-by columns are matched, we can use aggregated document
185185
if (remainingPredicateColumns.isEmpty() && remainingGroupByColumns.isEmpty()) {
186-
adder = docsWithField.grow(1);
187186
int docId = starTreeNode.getAggregatedDocId();
188-
adder.add(docId);
187+
docIds.add(docId);
189188
docNum = docId > docNum ? docId : docNum;
190189
continue;
191190
}
@@ -197,8 +196,7 @@ private StarTreeResult traverseStarTree() throws IOException {
197196
// remaining predicate columns for this node
198197
if (starTreeNode.isLeaf()) {
199198
for (long i = starTreeNode.getStartDocId(); i < starTreeNode.getEndDocId(); i++) {
200-
adder = docsWithField.grow(1);
201-
adder.add((int) i);
199+
docIds.add((int)i);
202200
docNum = (int)i > docNum ? (int)i : docNum;
203201
}
204202
continue;
@@ -289,6 +287,10 @@ private StarTreeResult traverseStarTree() throws IOException {
289287
}
290288
}
291289

290+
adder = docsWithField.grow(docIds.size());
291+
for(int id : docIds) {
292+
adder.add(id);
293+
}
292294
return new StarTreeResult(
293295
docsWithField,
294296
globalRemainingPredicateColumns != null ? globalRemainingPredicateColumns : Collections.emptySet(),

‎server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQueryBuilder.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ public static StarTreeQueryBuilder fromXContent(XContentParser parser) {
144144
protected Query doToQuery(QueryShardContext context) {
145145
// TODO : star tree supports either group by or filter
146146
if (predicateMap.size() > 0) {
147-
logger.info("Predicates: {} ", this.groupBy.toString() );
147+
//logger.info("Predicates: {} ", this.groupBy.toString() );
148148
return new StarTreeQuery(predicateMap, new HashSet<>());
149149
}
150150
logger.info("Group by : {} ", this.groupBy.toString() );

‎server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@
88

99
package org.opensearch.search.aggregations.bucket.startree;
1010

11+
import java.util.concurrent.atomic.AtomicBoolean;
12+
import java.util.concurrent.atomic.AtomicReference;
13+
import java.util.function.Predicate;
1114
import org.apache.logging.log4j.LogManager;
1215
import org.apache.logging.log4j.Logger;
1316
import org.apache.lucene.index.LeafReaderContext;
@@ -172,25 +175,30 @@ public InternalAggregation buildEmptyAggregation() {
172175
@Override
173176
protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
174177
StarTreeAggregatedValues values = (StarTreeAggregatedValues) ctx.reader().getAggregatedDocValues();
178+
final AtomicReference<StarTreeAggregatedValues> aggrVal = new AtomicReference<>(null);
175179
return new LeafBucketCollectorBase(sub, values) {
176180
@Override
177181
public void collect(int doc, long bucket) throws IOException {
178-
StarTreeAggregatedValues aggrVals = (StarTreeAggregatedValues) ctx.reader().getAggregatedDocValues();
179-
180-
Map<String, SortedNumericDocValues> fieldColToDocValuesMap = new HashMap<>();
182+
if(aggrVal.get() == null) {
183+
aggrVal.set((StarTreeAggregatedValues) ctx.reader().getAggregatedDocValues());
184+
}
185+
StarTreeAggregatedValues aggrVals = aggrVal.get();
186+
List<SortedNumericDocValues> fieldColToDocValuesMap = new ArrayList<>();
181187

182188
// TODO : validations
183189
for (String field : fieldCols) {
184-
fieldColToDocValuesMap.put(field, aggrVals.dimensionValues.get(field));
190+
fieldColToDocValuesMap.add(aggrVals.dimensionValues.get(field));
185191
}
186192
// Another hardcoding
187193
SortedNumericDocValues dv = aggrVals.metricValues.get(metrics.get(0));
188194
if (dv.advanceExact(doc)) {
189-
195+
long val1 = dv.nextValue();
190196
String key = getKey(fieldColToDocValuesMap, doc);
191-
197+
if(key.equals("") ) {
198+
return;
199+
}
192200
if (indexMap.containsKey(key)) {
193-
sumMap.put(key, sumMap.getOrDefault(key, 0l) + dv.nextValue());
201+
sumMap.put(key, sumMap.getOrDefault(key, 0l) + val1);
194202
} else {
195203
indexMap.put(key, indexMap.size());
196204
sumMap.put(key, dv.nextValue());
@@ -202,11 +210,11 @@ public void collect(int doc, long bucket) throws IOException {
202210

203211
}
204212

205-
private String getKey(Map<String, SortedNumericDocValues> fieldColsMap, int doc) throws IOException {
213+
private String getKey(List<SortedNumericDocValues> colsList, int doc) throws IOException {
206214
StringJoiner sj = new StringJoiner("-");
207-
for (Map.Entry<String, SortedNumericDocValues> fieldEntry : fieldColsMap.entrySet()) {
208-
fieldEntry.getValue().advanceExact(doc);
209-
long val = fieldEntry.getValue().nextValue();
215+
for (SortedNumericDocValues col : colsList) {
216+
col.advanceExact(doc);
217+
long val = col.nextValue();
210218
//System.out.println("Key field : " + fieldEntry.getKey() + " Value : " + val);
211219
sj.add("" + val);
212220
}

‎server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131

3232
package org.opensearch.search.aggregations.metrics;
3333

34+
import org.apache.logging.log4j.LogManager;
35+
import org.apache.logging.log4j.Logger;
3436
import org.apache.lucene.index.LeafReaderContext;
3537
import org.apache.lucene.search.ScoreMode;
3638
import org.opensearch.common.lease.Releasables;
@@ -42,6 +44,7 @@
4244
import org.opensearch.search.aggregations.InternalAggregation;
4345
import org.opensearch.search.aggregations.LeafBucketCollector;
4446
import org.opensearch.search.aggregations.LeafBucketCollectorBase;
47+
import org.opensearch.search.aggregations.bucket.startree.StarTreeAggregator;
4548
import org.opensearch.search.aggregations.support.ValuesSource;
4649
import org.opensearch.search.aggregations.support.ValuesSourceConfig;
4750
import org.opensearch.search.internal.SearchContext;
@@ -61,6 +64,8 @@ public class SumAggregator extends NumericMetricsAggregator.SingleValue {
6164

6265
private DoubleArray sums;
6366
private DoubleArray compensations;
67+
private static final Logger logger = LogManager.getLogger(SumAggregator.class);
68+
6469

6570
SumAggregator(
6671
String name,
@@ -95,11 +100,13 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc
95100
return new LeafBucketCollectorBase(sub, values) {
96101
@Override
97102
public void collect(int doc, long bucket) throws IOException {
103+
//logger.info("collecting doc : {}", doc);
98104
sums = bigArrays.grow(sums, bucket + 1);
99105
compensations = bigArrays.grow(compensations, bucket + 1);
100106

101107
if (values.advanceExact(doc)) {
102108
final int valuesCount = values.docValueCount();
109+
//logger.info("values count : {}" , valuesCount);
103110
// Compute the sum of double values with Kahan summation algorithm which is more
104111
// accurate than naive summation.
105112
double sum = sums.get(bucket);

0 commit comments

Comments
 (0)