Skip to content

Commit 7c8c924

Browse files
yabspyabsp
committed
Add QueryBuilders for PostgreSQL native queries
1 parent d3b49c7 commit 7c8c924

6 files changed

Lines changed: 736 additions & 0 deletions

File tree

‎src/main/java/org/polypheny/simpleclient/scenario/vectorbench/PgDataGenerator.java‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
/*
2+
* The MIT License (MIT)
3+
*
4+
* Copyright (c) 2019-5/26/26, 5:48 PM The Polypheny Project
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a
7+
* copy of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
*/
24+
25+
package org.polypheny.simpleclient.scenario.vectorbench;
26+
27+
import lombok.extern.slf4j.Slf4j;
28+
import org.polypheny.simpleclient.executor.Executor;
29+
import org.polypheny.simpleclient.executor.ExecutorException;
30+
import org.polypheny.simpleclient.main.ProgressReporter;
31+
import org.polypheny.simpleclient.query.BatchableInsert;
32+
import org.polypheny.simpleclient.query.RawQuery;
33+
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.dml.PgInsertRealFeature;
34+
import java.util.LinkedList;
35+
import java.util.List;
36+
37+
38+
@Slf4j
39+
public class PgDataGenerator {
40+
41+
private final Executor executor;
42+
private final VectorBenchConfig config;
43+
private final ProgressReporter progressReporter;
44+
private final List<BatchableInsert> batch;
45+
private boolean aborted;
46+
47+
48+
PgDataGenerator( Executor executor, VectorBenchConfig config, ProgressReporter progressReporter ) {
49+
this.executor = executor;
50+
this.config = config;
51+
this.progressReporter = progressReporter;
52+
this.batch = new LinkedList<>();
53+
this.aborted = false;
54+
}
55+
56+
57+
void generateRealFeatures() throws ExecutorException {
58+
PgInsertRealFeature builder = new PgInsertRealFeature( config.randomSeedInsert, config.dimensionFeatureVectors );
59+
for ( int i = 0; i < config.numberOfEntries; i++ ) {
60+
if ( aborted ) break;
61+
addToBatch( builder.getNewQuery() );
62+
progressReporter.update( 1 );
63+
}
64+
flushBatch();
65+
}
66+
67+
68+
private void addToBatch( BatchableInsert query ) throws ExecutorException {
69+
batch.add( query );
70+
if ( batch.size() >= config.batchSizeInserts ) {
71+
flushBatch();
72+
}
73+
}
74+
75+
76+
private void flushBatch() throws ExecutorException {
77+
if ( batch.isEmpty() ) return;
78+
StringBuilder sb = new StringBuilder();
79+
boolean first = true;
80+
for ( BatchableInsert insert : batch ) {
81+
if ( first ) {
82+
sb.append( insert.getSql() );
83+
first = false;
84+
} else {
85+
sb.append( "," ).append( insert.getSqlRowExpression() );
86+
}
87+
}
88+
executor.executeQuery( RawQuery.builder().sql( sb.toString() ).expectResultSet( false ).build() );
89+
executor.executeCommit();
90+
batch.clear();
91+
}
92+
93+
94+
95+
public void abort() {
96+
aborted = true;
97+
}
98+
}
99+

‎src/main/java/org/polypheny/simpleclient/scenario/vectorbench/PgVectorBench.java‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
/*
2+
* The MIT License (MIT)
3+
*
4+
* Copyright (c) 2019-5/26/26, 5:48 PM The Polypheny Project
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a
7+
* copy of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
*/
24+
25+
package org.polypheny.simpleclient.scenario.vectorbench;
26+
27+
import lombok.extern.slf4j.Slf4j;
28+
import org.polypheny.simpleclient.QueryMode;
29+
import org.polypheny.simpleclient.executor.Executor;
30+
import org.polypheny.simpleclient.executor.Executor.DatabaseInstance;
31+
import org.polypheny.simpleclient.executor.ExecutorException;
32+
import org.polypheny.simpleclient.main.CsvWriter;
33+
import org.polypheny.simpleclient.main.ProgressReporter;
34+
import org.polypheny.simpleclient.query.Query;
35+
import org.polypheny.simpleclient.query.QueryBuilder;
36+
import org.polypheny.simpleclient.query.QueryListEntry;
37+
import org.polypheny.simpleclient.query.RawQuery;
38+
import org.polypheny.simpleclient.scenario.PolyphenyScenario;
39+
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.ddl.PgCreateRealFeature;
40+
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.dql.PgSimpleKnnRealFeature;
41+
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.dql.PgSimpleKnnRealFeatureFiltered;
42+
import java.io.File;
43+
import java.util.Collections;
44+
import java.util.LinkedList;
45+
import java.util.List;
46+
import java.util.Random;
47+
import java.util.Vector;
48+
49+
50+
@Slf4j
51+
public class PgVectorBench extends PolyphenyScenario {
52+
53+
private final VectorBenchConfig config;
54+
55+
56+
public PgVectorBench( Executor.ExecutorFactory executorFactory, VectorBenchConfig config,
57+
boolean commitAfterEveryQuery, boolean dumpQueryList ) {
58+
super( executorFactory, commitAfterEveryQuery, dumpQueryList, QueryMode.TABLE );
59+
this.config = config;
60+
}
61+
62+
63+
@Override
64+
public void createSchema( DatabaseInstance databaseInstance, boolean includingKeys ) {
65+
log.info( "Creating schema..." );
66+
Executor executor = null;
67+
try {
68+
executor = executorFactory.createExecutorInstance();
69+
executor.executeQuery( new RawQuery( "CREATE EXTENSION IF NOT EXISTS vector", null, false ) );
70+
executor.executeQuery( new PgCreateRealFeature( config.dimensionFeatureVectors ).getNewQuery() );
71+
} catch ( ExecutorException e ) {
72+
throw new RuntimeException( "Exception while creating schema", e );
73+
} finally {
74+
commitAndCloseExecutor( executor );
75+
}
76+
}
77+
78+
79+
@Override
80+
public void generateData( DatabaseInstance databaseInstance, ProgressReporter progressReporter ) {
81+
log.info( "Generating data..." );
82+
Executor executor = executorFactory.createExecutorInstance();
83+
PgDataGenerator dataGenerator = new PgDataGenerator( executor, config, progressReporter );
84+
try {
85+
dataGenerator.generateRealFeatures();
86+
} catch ( ExecutorException e ) {
87+
throw new RuntimeException( "Exception while generating data", e );
88+
} finally {
89+
commitAndCloseExecutor( executor );
90+
}
91+
}
92+
93+
94+
@Override
95+
public long execute( ProgressReporter progressReporter, CsvWriter csvWriter, File outputDirectory, int numberOfThreads ) {
96+
log.info( "Preparing query list..." );
97+
List<QueryListEntry> queryList = new Vector<>();
98+
addNumberOfTimes( queryList, new PgSimpleKnnRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm ), config.numberOfSimpleKnnRealFeatureQueries );
99+
addNumberOfTimes( queryList, new PgSimpleKnnRealFeatureFiltered( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm, "cat_A" ), config.numberOfSimpleKnnRealFeatureFilteredQueries );
100+
return commonExecute( queryList, progressReporter, outputDirectory, numberOfThreads,
101+
Query::getSql, () -> executorFactory.createExecutorInstance( csvWriter ), new Random() );
102+
}
103+
104+
105+
@Override
106+
public void warmUp( ProgressReporter progressReporter ) {
107+
log.info( "Warm-up..." );
108+
PgSimpleKnnRealFeature knnBuilder = new PgSimpleKnnRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
109+
Executor executor = null;
110+
for ( int i = 0; i < config.numberOfWarmUpIterations; i++ ) {
111+
try {
112+
executor = executorFactory.createExecutorInstance();
113+
if ( config.numberOfSimpleKnnRealFeatureQueries > 0 ) {
114+
executor.executeQuery( knnBuilder.getNewQuery() );
115+
}
116+
} catch ( ExecutorException e ) {
117+
throw new RuntimeException( "Error during warm-up", e );
118+
} finally {
119+
commitAndCloseExecutor( executor );
120+
}
121+
try {
122+
Thread.sleep( 10000 );
123+
} catch ( InterruptedException e ) {
124+
throw new RuntimeException( "Interrupted during warm-up", e );
125+
}
126+
}
127+
}
128+
129+
130+
@Override
131+
public int getNumberOfInsertThreads() {
132+
return 1;
133+
}
134+
135+
136+
private void addNumberOfTimes( List<QueryListEntry> list, QueryBuilder builder, int count ) {
137+
int id = queryTypes.size() + 1;
138+
queryTypes.put( id, builder.getNewQuery().getSql() );
139+
measuredTimePerQueryType.put( id, Collections.synchronizedList( new LinkedList<>() ) );
140+
for ( int i = 0; i < count; i++ ) {
141+
list.add( new QueryListEntry( builder.getNewQuery(), id ) );
142+
}
143+
}
144+
}
145+

‎src/main/java/org/polypheny/simpleclient/scenario/vectorbench/queryBuilder/postgres/ddl/PgCreateRealFeature.java‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* The MIT License (MIT)
3+
*
4+
* Copyright (c) 2019-2026 The Polypheny Project
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a
7+
* copy of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
*/
24+
25+
package org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.ddl;
26+
27+
import kong.unirest.core.HttpRequest;
28+
import org.apache.commons.lang3.tuple.ImmutablePair;
29+
import org.polypheny.simpleclient.query.Query;
30+
import org.polypheny.simpleclient.query.QueryBuilder;
31+
import java.util.Map;
32+
33+
34+
public class PgCreateRealFeature extends QueryBuilder {
35+
36+
private final int dimension;
37+
38+
39+
public PgCreateRealFeature( int dimension ) {
40+
this.dimension = dimension;
41+
}
42+
43+
44+
@Override
45+
public Query getNewQuery() {
46+
return new PgCreateRealFeatureQuery( dimension );
47+
}
48+
49+
50+
private static class PgCreateRealFeatureQuery extends Query {
51+
52+
private final int dimension;
53+
54+
55+
PgCreateRealFeatureQuery( int dimension ) {
56+
super( false );
57+
this.dimension = dimension;
58+
}
59+
60+
61+
@Override
62+
public String getSql() {
63+
return "CREATE TABLE knn_realfeature ("
64+
+ "id INTEGER NOT NULL, "
65+
+ "category VARCHAR(50), "
66+
+ "feature vector(" + dimension + ") NOT NULL, "
67+
+ "PRIMARY KEY (id))";
68+
}
69+
70+
71+
@Override
72+
public String getParameterizedSqlQuery() {
73+
return null;
74+
}
75+
76+
77+
@Override
78+
public Map<Integer, ImmutablePair<DataTypes, Object>> getParameterValues() {
79+
return null;
80+
}
81+
82+
83+
@Override
84+
public HttpRequest<?> getRest() {
85+
return null;
86+
}
87+
88+
89+
@Override
90+
public String getMongoQl() {
91+
return null;
92+
}
93+
94+
}
95+
96+
}

0 commit comments

Comments
 (0)