Write recall summary to file on CLI path of vectorbench

Author: yabsp

.gitignore

@@ -256,9 +256,11 @@ nbdist/
 # Avoid ignoring PolySqlParser jar file (.jar files are usually ignored)
 !/libs/PolySqlParser-1.0.jar
 
+# vectorbench CLI path
 /results.csv
 /results-pg.csv
 /recall-groundtruth.csv
+/recall.csv
 
 # humble video libraries
 libhumblevideo-0.dll

src/main/java/org/polypheny/simpleclient/main/VectorBenchScenario.java

@@ -37,6 +37,7 @@
 import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.dql.SimpleKnnRealFeature;
 import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.postgres.dql.PgSimpleKnnRealFeature;
 import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.util.Objects;
 import java.util.Properties;
@@ -117,7 +118,8 @@ private static void runRecall( ExecutorFactory executorFactory, VectorBenchConfi
             if ( capture ) {
                 evaluator.captureGroundTruth();
             } else {
-                evaluator.evaluate();
+                double recall = evaluator.evaluate();
+                writeRecall( config, recall );
             }
         } finally {
             try {
@@ -129,6 +131,17 @@ private static void runRecall( ExecutorFactory executorFactory, VectorBenchConfi
     }
 
 
+    /** Writes the recall@k of the last `recall` run to recall.csv in the working directory. */
+    private static void writeRecall( VectorBenchConfig config, double recall ) {
+        try ( FileWriter fw = new FileWriter( "recall.csv" ) ) {
+            fw.write( "k,recall\n" );
+            fw.write( config.limitKnnQueries + "," + recall + "\n" );
+        } catch ( IOException e ) {
+            log.error( "Could not write recall.csv", e );
+        }
+    }
+
+
     public static void data( ExecutorFactory executorFactory, int multiplier, boolean commitAfterEveryQuery ) {
         VectorBenchConfig config = new VectorBenchConfig( getProperties(), multiplier );
         VectorBench vectorBench = new VectorBench( executorFactory, config, commitAfterEveryQuery, false );

src/main/java/org/polypheny/simpleclient/scenario/vectorbench/VectorBench.java

@@ -43,6 +43,7 @@
 import org.polypheny.simpleclient.scenario.PolyphenyScenario;
 import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.dql.SimpleKnnBooleanFeature;
 import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.dql.SimpleKnnBooleanFeatureFiltered;
+import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.dql.SimpleKnnIdIntFeature;
 import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.dql.SimpleKnnRealFeatureFiltered;
 import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.ddl.CreateBooleanFeature;
 import org.polypheny.simpleclient.scenario.vectorbench.queryBuilder.ddl.CreateIntFeature;
@@ -192,6 +193,9 @@ public long execute( ProgressReporter progressReporter, CsvWriter csvWriter, Fil
         addNumberOfTimes( queryList, new SimpleKnnRealFeatureFiltered( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm, "cat_A" ), config.numberOfSimpleKnnRealFeatureFilteredQueries );
         addNumberOfTimes( queryList, new SimpleKnnBooleanFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.booleanDistanceNorm ), config.numberOfSimpleKnnBooleanFeatureQueries );
         addNumberOfTimes( queryList, new SimpleKnnBooleanFeatureFiltered( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.booleanDistanceNorm, "cat_A" ), config.numberOfSimpleKnnBooleanFeatureFilteredQueries );
+        addNumberOfTimes( queryList, new MetadataKnnRealCrossJoin( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm ), config.numberOfMetadataKnnRealCrossJoinQueries );
+        addNumberOfTimes( queryList, new SimpleKnnIdIntFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm ), config.numberOfSimpleKnnIdIntFeatureQueries );
+
 
 
         return commonExecute( queryList, progressReporter, outputDirectory, numberOfThreads, Query::getSql, () -> executorFactory.createExecutorInstance( csvWriter ), new Random() );
@@ -214,7 +218,7 @@ public void warmUp( ProgressReporter progressReporter ) {
         SimpleKnnRealFeatureFiltered simpleKnnRealFeatureFiltered = new SimpleKnnRealFeatureFiltered( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.distanceNorm, "cat_A" );
         SimpleKnnBooleanFeature simpleKnnBooleanFeature = new SimpleKnnBooleanFeature( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.booleanDistanceNorm );
         SimpleKnnBooleanFeatureFiltered simpleKnnBooleanFeatureFiltered = new SimpleKnnBooleanFeatureFiltered( config.randomSeedQuery, config.dimensionFeatureVectors, config.limitKnnQueries, config.booleanDistanceNorm, "cat_A" );
-
+        SimpleKnnIdIntFeature simpleKnnIdIntFeature = new SimpleKnnIdIntFeature( config.randomSeedQuery, config.dimensionFeatureVectors,config.limitKnnQueries, config.distanceNorm );
 
         for ( int i = 0; i < config.numberOfWarmUpIterations; i++ ) {
             try {
@@ -234,6 +238,9 @@ public void warmUp( ProgressReporter progressReporter ) {
                 if ( config.numberOfMetadataKnnIntFeatureQueries > 0 ) {
                     executor.executeQuery( metadataKnnIntFeature.getNewQuery() );
                 }
+                if ( config.numberOfSimpleKnnIdIntFeatureQueries > 0 ) {
+                    executor.executeQuery( simpleKnnIdIntFeature.getNewQuery() );
+                }
                 if ( config.numberOfMetadataKnnRealFeatureQueries > 0 ) {
                     executor.executeQuery( metadataKnnRealFeature.getNewQuery() );
                 }

src/main/resources/org/polypheny/simpleclient/scenario/vectorbench/vector.properties

@@ -3,16 +3,16 @@ scenario = "vectorBench"
 # Execution of queries on: polypheny | postgres
 mode = polypheny
 
-dataStoreFeature = postgresql
+dataStoreFeature = postgresql1
 dataStoreMeta = hsqldb
 
 # Default, pgvector, PostGIS, pgvector & PostGIS
-postgresImageVariant = Default
+postgresImageVariant = pgvector & PostGIS
 
 # true => DDL: REAL NOT NULL ARRAY(1,3)
 # false => DDL: REAL ARRAY(1,3)
 # Note: Ignored by direct postgres path
-supportsNotNullArray = false
+supportsNotNullArray = true
 
 # False -> Does not create index
 # Note: When run in cli, vector index will not create indexes if set to false
@@ -27,21 +27,22 @@ queryEfSearch = 40
 queryProbes = 1
 # ---------------------------------------------------------------------------------------
 
+# Number of sampled query vectors used to compute recall@k
+# Note: CLI path only
+numberOfRecallQueries = 100
+
 # Vector index settings,
 # Method: hnsw or ivfflat
-indexMethod = ivfflat
+indexMethod = hnsw
 # Only used for HNSW index
 indexM = 16
 indexEfConstruction = 64
 # Only used for IVFFlat index
 indexLists = 100
 
-# Number of sampled query vectors used to compute recall@k
-numberOfRecallQueries = 100
-
 # PostgreSQL direct connect settings
 # Used when mode = postgres
-postgresHost = 127.0.0.1
+postgresHost = localhost
 
 numberOfThreads = 4
 progressReportBase = 100
@@ -65,19 +66,19 @@ numberOfSimpleKnnIntFeatureQueries = 0
 numberOfSimpleKnnIdIntFeatureQueries = 0
 numberOfMetadataKnnIntFeatureQueries = 0
 
-numberOfSimpleMetadataQueries = 10
+numberOfSimpleMetadataQueries = 0
 
 # Real (float vector mapping)
-numberOfSimpleKnnRealFeatureQueries = 10
-numberOfSimpleKnnIdRealFeatureQueries = 10
-numberOfMetadataKnnRealFeatureQueries = 10
-numberOfSimpleKnnRealCrossJoinQueries = 10
-numberOfMetadataKnnRealCrossJoinQueries = 10
-numberOfSimpleKnnRealFeatureFilteredQueries = 10
+numberOfSimpleKnnRealFeatureQueries = 0
+numberOfSimpleKnnIdRealFeatureQueries = 0
+numberOfMetadataKnnRealFeatureQueries = 0
+numberOfSimpleKnnRealCrossJoinQueries = 0
+numberOfMetadataKnnRealCrossJoinQueries = 0
+numberOfSimpleKnnRealFeatureFilteredQueries = 0
 
 # Boolean (bit vector mapping)
 numberOfSimpleKnnBooleanFeatureFilteredQueries = 10
-numberOfSimpleKnnBooleanFeatureQueries = 10
+numberOfSimpleKnnBooleanFeatureQueries = 0
 
 
 limitKnnQueries = 10