Skip to content

Commit 14e9cd1

Browse files
yabspyabsp
committed
Integrate new PostgreSQL QueryBuilders
1 parent 8eaa0e7 commit 14e9cd1

4 files changed

Lines changed: 97 additions & 7 deletions

File tree

‎src/main/java/org/polypheny/simpleclient/scenario/vectorbench/PgDataGenerator.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.polypheny.simpleclient.main.ProgressReporter;
3131
import org.polypheny.simpleclient.query.BatchableInsert;
3232
import org.polypheny.simpleclient.query.RawQuery;
33+
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.dml.PgInsertBooleanFeature;
3334
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.dml.PgInsertRealFeature;
3435
import java.util.LinkedList;
3536
import java.util.List;
@@ -65,6 +66,17 @@ void generateRealFeatures() throws ExecutorException {
6566
}
6667

6768

69+
void generateBooleanFeatures() throws ExecutorException {
70+
PgInsertBooleanFeature builder = new PgInsertBooleanFeature( config.randomSeedInsert, config.dimensionFeatureVectors );
71+
for ( int i = 0; i < config.numberOfEntries; i++ ) {
72+
if ( aborted ) break;
73+
addToBatch( builder.getNewQuery() );
74+
progressReporter.update( 1 );
75+
}
76+
flushBatch();
77+
}
78+
79+
6880
private void addToBatch( BatchableInsert query ) throws ExecutorException {
6981
batch.add( query );
7082
if ( batch.size() >= config.batchSizeInserts ) {

‎src/main/java/org/polypheny/simpleclient/scenario/vectorbench/PgVectorBench.java‎

Lines changed: 57 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,14 @@
3636
import org.polypheny.simpleclient.query.QueryListEntry;
3737
import org.polypheny.simpleclient.query.RawQuery;
3838
import org.polypheny.simpleclient.scenario.PolyphenyScenario;
39+
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.ddl.PgCreateBooleanFeature;
40+
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.ddl.PgCreateBooleanFeatureIndex;
3941
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.ddl.PgCreateRealFeature;
4042
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.ddl.PgCreateRealFeatureIndex;
43+
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.dql.PgSimpleKnnBooleanFeature;
44+
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.dql.PgSimpleKnnBooleanFeatureFiltered;
45+
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.dql.PgSimpleKnnIdRealFeature;
46+
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.dql.PgSimpleKnnRealCrossJoin;
4147
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.dql.PgSimpleKnnRealFeature;
4248
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.dql.PgSimpleKnnRealFeatureFiltered;
4349
import java.io.File;
@@ -69,6 +75,7 @@ public void createSchema( DatabaseInstance databaseInstance, boolean includingKe
6975
executor = executorFactory.createExecutorInstance();
7076
executor.executeQuery( new RawQuery( "CREATE EXTENSION IF NOT EXISTS vector", null, false ) );
7177
executor.executeQuery( new PgCreateRealFeature( config.dimensionFeatureVectors ).getNewQuery() );
78+
executor.executeQuery( new PgCreateBooleanFeature( config.dimensionFeatureVectors ).getNewQuery() );
7279
} catch ( ExecutorException e ) {
7380
throw new RuntimeException( "Exception while creating schema", e );
7481
} finally {
@@ -85,10 +92,19 @@ public void createIndex() {
8592
try {
8693
executor = executorFactory.createExecutorInstance();
8794
long start = System.nanoTime();
88-
executor.executeQuery( new PgCreateRealFeatureIndex( config.indexMethod, config.distanceNorm, config.indexM, config.indexEfConstruction, config.indexLists ).getNewQuery() );
95+
if ( indexSupportsMetric( config.indexMethod, config.distanceNorm ) ) {
96+
executor.executeQuery( new PgCreateRealFeatureIndex( config.indexMethod, config.distanceNorm, config.indexM, config.indexEfConstruction, config.indexLists ).getNewQuery() );
97+
} else {
98+
log.info( "Skipping real index: {} does not support metric '{}'.", config.indexMethod, config.distanceNorm );
99+
}
100+
if ( indexSupportsMetric( config.indexMethod, config.booleanDistanceNorm ) ) {
101+
executor.executeQuery( new PgCreateBooleanFeatureIndex( config.indexMethod, config.booleanDistanceNorm, config.indexM, config.indexEfConstruction, config.indexLists ).getNewQuery() );
102+
} else {
103+
log.info( "Skipping boolean index: {} does not support metric '{}'.", config.indexMethod, config.booleanDistanceNorm );
104+
}
89105
executor.executeCommit();
90106
long durationMillis = ( System.nanoTime() - start ) / 1_000_000L;
91-
log.info( "Vector index built in {} ms", durationMillis );
107+
log.info( "Vector indexes built in {} ms", durationMillis );
92108

93109
String conf = config.indexMethod.equals( "hnsw" )
94110
? "hnsw.ef_search = " + config.queryEfSearch
@@ -111,6 +127,7 @@ public void generateData( DatabaseInstance databaseInstance, ProgressReporter pr
111127
PgDataGenerator dataGenerator = new PgDataGenerator( executor, config, progressReporter );
112128
try {
113129
dataGenerator.generateRealFeatures();
130+
dataGenerator.generateBooleanFeatures();
114131
} catch ( ExecutorException e ) {
115132
throw new RuntimeException( "Exception while generating data", e );
116133
} finally {
@@ -130,6 +147,11 @@ public long execute( ProgressReporter progressReporter, CsvWriter csvWriter, Fil
130147
List<QueryListEntry> queryList = new Vector<>();
131148
addNumberOfTimes( queryList, new PgSimpleKnnRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm ), config.numberOfSimpleKnnRealFeatureQueries );
132149
addNumberOfTimes( queryList, new PgSimpleKnnRealFeatureFiltered( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm, "cat_A" ), config.numberOfSimpleKnnRealFeatureFilteredQueries );
150+
addNumberOfTimes( queryList, new PgSimpleKnnIdRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm ), config.numberOfSimpleKnnIdRealFeatureQueries );
151+
addNumberOfTimes( queryList, new PgSimpleKnnRealCrossJoin( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm ), config.numberOfSimpleKnnRealCrossJoinQueries );
152+
addNumberOfTimes( queryList, new PgSimpleKnnBooleanFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.booleanDistanceNorm ), config.numberOfSimpleKnnBooleanFeatureQueries );
153+
addNumberOfTimes( queryList, new PgSimpleKnnBooleanFeatureFiltered( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.booleanDistanceNorm, "cat_A" ), config.numberOfSimpleKnnBooleanFeatureFilteredQueries );
154+
133155
return commonExecute( queryList, progressReporter, outputDirectory, numberOfThreads,
134156
Query::getSql, () -> executorFactory.createExecutorInstance( csvWriter ), new Random() );
135157
}
@@ -138,14 +160,36 @@ public long execute( ProgressReporter progressReporter, CsvWriter csvWriter, Fil
138160
@Override
139161
public void warmUp( ProgressReporter progressReporter ) {
140162
log.info( "Warm-up..." );
141-
PgSimpleKnnRealFeature knnBuilder = new PgSimpleKnnRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
142163
Executor executor = null;
164+
PgSimpleKnnRealFeature pgSimpleKnnRealFeature = new PgSimpleKnnRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
165+
PgSimpleKnnRealFeatureFiltered pgSimpleKnnRealFeatureFiltered = new PgSimpleKnnRealFeatureFiltered( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm, "cat_A" );
166+
PgSimpleKnnRealCrossJoin pgSimpleKnnRealCrossJoin = new PgSimpleKnnRealCrossJoin( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
167+
PgSimpleKnnIdRealFeature pgSimpleKnnIdRealFeature = new PgSimpleKnnIdRealFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm );
168+
PgSimpleKnnBooleanFeature pgSimpleKnnBooleanFeature = new PgSimpleKnnBooleanFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.booleanDistanceNorm );
169+
PgSimpleKnnBooleanFeatureFiltered pgSimpleKnnBooleanFeatureFiltered = new PgSimpleKnnBooleanFeatureFiltered( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.booleanDistanceNorm, "cat_A" );
170+
143171
for ( int i = 0; i < config.numberOfWarmUpIterations; i++ ) {
144172
try {
145173
executor = executorFactory.createExecutorInstance();
146174
if ( config.numberOfSimpleKnnRealFeatureQueries > 0 ) {
147-
executor.executeQuery( knnBuilder.getNewQuery() );
175+
executor.executeQuery( pgSimpleKnnRealFeature.getNewQuery() );
176+
}
177+
if ( config.numberOfSimpleKnnRealFeatureFilteredQueries > 0 ) {
178+
executor.executeQuery( pgSimpleKnnRealFeatureFiltered.getNewQuery() );
179+
}
180+
if ( config.numberOfSimpleKnnRealCrossJoinQueries > 0 ) {
181+
executor.executeQuery( pgSimpleKnnRealCrossJoin.getNewQuery() );
182+
}
183+
if ( config.numberOfSimpleKnnIdRealFeatureQueries > 0 ) {
184+
executor.executeQuery( pgSimpleKnnIdRealFeature.getNewQuery() );
148185
}
186+
if ( config.numberOfSimpleKnnBooleanFeatureQueries > 0 ) {
187+
executor.executeQuery( pgSimpleKnnBooleanFeature.getNewQuery() );
188+
}
189+
if ( config.numberOfSimpleKnnBooleanFeatureFilteredQueries > 0 ) {
190+
executor.executeQuery( pgSimpleKnnBooleanFeatureFiltered.getNewQuery() );
191+
}
192+
149193
} catch ( ExecutorException e ) {
150194
throw new RuntimeException( "Error during warm-up", e );
151195
} finally {
@@ -166,6 +210,15 @@ public int getNumberOfInsertThreads() {
166210
}
167211

168212

213+
private static boolean indexSupportsMetric( String method, String metric ) {
214+
// HNSW supports all metrics; IVFFlat does not support L1 or JACCARD.
215+
if ( method.equalsIgnoreCase( "ivfflat" ) ) {
216+
return !( metric.equalsIgnoreCase( "L1" ) || metric.equalsIgnoreCase( "JACCARD" ) );
217+
}
218+
return true;
219+
}
220+
221+
169222
private void addNumberOfTimes( List<QueryListEntry> list, QueryBuilder builder, int count ) {
170223
int id = queryTypes.size() + 1;
171224
queryTypes.put( id, builder.getNewQuery().getSql() );

‎src/main/java/org/polypheny/simpleclient/scenario/vectorbench/VectorBench.java‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.dql.SimpleKnnRealFeatureFiltered;
4747
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.ddl.CreateBooleanFeature;
4848
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.ddl.CreateIntFeature;
49+
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.ddl.CreateBooleanFeatureIndex;
4950
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.ddl.CreateMetadata;
5051
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.ddl.CreateRealFeature;
5152
import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.ddl.CreateRealFeatureIndex;
@@ -110,14 +111,29 @@ public void createIndex() {
110111
if ( !config.useIndex ) {
111112
return;
112113
}
114+
// Polypheny vector-maps array columns only when declared with element NOT NULL (REAL NOT NULL ARRAY).
115+
// A plain ARRAY column is not vector-mapped, so no vector index can be built.
116+
if ( !config.supportsNotNullArray ) {
117+
log.info( "Skipping index creation: plain ARRAY columns are not vector-mapped in Polypheny (requires NOT NULL ARRAY)." );
118+
return;
119+
}
113120
Executor executor = null;
114121
try {
115122
executor = executorFactory.createExecutorInstance();
116123
long start = System.nanoTime();
117-
executor.executeQuery( new CreateRealFeatureIndex( featureStore, config.indexMethod, config.distanceNorm, config.indexM, config.indexEfConstruction, config.indexLists ).getNewQuery() );
124+
if ( indexSupportsMetric( config.indexMethod, config.distanceNorm ) ) {
125+
executor.executeQuery( new CreateRealFeatureIndex( featureStore, config.indexMethod, config.distanceNorm, config.indexM, config.indexEfConstruction, config.indexLists ).getNewQuery() );
126+
} else {
127+
log.info( "Skipping real index: {} does not support metric '{}'.", config.indexMethod, config.distanceNorm );
128+
}
129+
if ( indexSupportsMetric( config.indexMethod, config.booleanDistanceNorm ) ) {
130+
executor.executeQuery( new CreateBooleanFeatureIndex( featureStore, config.indexMethod, config.booleanDistanceNorm, config.indexM, config.indexEfConstruction, config.indexLists ).getNewQuery() );
131+
} else {
132+
log.info( "Skipping boolean index: {} does not support metric '{}'.", config.indexMethod, config.booleanDistanceNorm );
133+
}
118134
executor.executeCommit();
119135
long durationMillis = ( System.nanoTime() - start ) / 1_000_000L;
120-
log.info( "Vector index built in {} ms", durationMillis );
136+
log.info( "Vector indexes built in {} ms", durationMillis );
121137
} catch ( ExecutorException e ) {
122138
throw new RuntimeException( "Exception while creating vector index", e );
123139
} finally {
@@ -126,6 +142,15 @@ public void createIndex() {
126142
}
127143

128144

145+
private static boolean indexSupportsMetric( String method, String metric ) {
146+
// HNSW supports all metrics, IVFFlat does not support L1 or JACCARD.
147+
if ( method.equalsIgnoreCase( "ivfflat" ) ) {
148+
return !( metric.equalsIgnoreCase( "L1" ) || metric.equalsIgnoreCase( "JACCARD" ) );
149+
}
150+
return true;
151+
}
152+
153+
129154
@Override
130155
public void generateData( DatabaseInstance databaseInstance, ProgressReporter progressReporter ) {
131156
log.info( "Generating data..." );

‎src/main/java/org/polypheny/simpleclient/scenario/vectorbench/VectorBenchConfig.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ public VectorBenchConfig(Map<String, String> cdl ) {
150150
dataStoreFeature = cdl.get( "dataStoreFeature" );
151151
dataStoreMetadata = cdl.get( "dataStoreMetadata" );
152152

153-
supportsNotNullArray = Boolean.parseBoolean( cdl.get( "supportsNotNullArray" ) );
153+
supportsNotNullArray = cdl.get( "supportsNotNullArray" ) == null || Boolean.parseBoolean( cdl.get( "supportsNotNullArray" ) );
154154

155155
if ( dataStoreFeature.equals( dataStoreMetadata ) ) {
156156
dataStores.add( dataStoreFeature );

0 commit comments

Comments
 (0)