Fix PQ file path and migrate legacy PQ files

Author: tae898

bindings/python/examples/benchmark-vector/summaries/arcadedb_msmarco_MSMARCO-100K.md

@@ -0,0 +1,5 @@
+# ArcadeDB MSMARCO MSMARCO-100K (1000 queries, Recall@50)
+
+| quantization   | store_vectors_in_graph   | add_hierarchy   |   max_connections |   beam_width |   overquery_factor |   batch_size |   load_corpus_s |   load_corpus_rss_mb |   ingest_s |   ingest_rss_mb |   create_index_s |   create_index_rss_mb |   build_graph_now_s |   build_graph_now_rss_mb |   warmup_s |   warmup_rss_mb |   search_s |   search_rss_mb |   recall@50_before_close |   close_db_s |   close_db_rss_mb |   open_db_s |   open_db_rss_mb |   warmup_after_reopen_s |   warmup_after_reopen_rss_mb |   search_after_reopen_s |   search_after_reopen_rss_mb |   recall@50_after_reopen |   peak_rss_mb |   db_size_mb | total_duration   |
+|:---------------|:-------------------------|:----------------|------------------:|-------------:|-------------------:|-------------:|----------------:|---------------------:|-----------:|----------------:|-----------------:|----------------------:|--------------------:|-------------------------:|-----------:|----------------:|-----------:|----------------:|-------------------------:|-------------:|------------------:|------------:|-----------------:|------------------------:|-----------------------------:|------------------------:|-----------------------------:|-------------------------:|--------------:|-------------:|:-----------------|
+| PRODUCT        | False                    | True            |                12 |           64 |                  1 |        10000 |               0 |                    0 |       6.96 |         1369.83 |            1.187 |               281.465 |              44.925 |                  1425.59 |      0.048 |           1.812 |      1.394 |          36.148 |                   0.8956 |        0.018 |            -0.113 |       0.205 |            16.23 |                   0.533 |                        0.074 |                   1.273 |                       309.82 |                   0.8956 |       3659.88 |      602.014 | 57.216s          |

engine/src/main/java/com/arcadedb/index/vector/LSMVectorIndex.java

@@ -50,6 +50,8 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -244,7 +246,7 @@ public LSMVectorIndex(final DatabaseInternal database, final String name, final
 
       // Create PQ file handler for Product Quantization (zero-disk-I/O search)
       // Note: PQ file uses direct I/O (not ArcadeDB pages) since it's loaded entirely into memory
-      this.pqFile = new LSMVectorIndexPQFile(filePath);
+      this.pqFile = createPQFileWithFallback(mutable.getFilePath());
 
       LogManager.instance()
               .log(this, Level.FINE, "Created LSMVectorIndex: indexName=%s, vectorFileId=%d, graphFileId=%d", indexName,
@@ -286,7 +288,7 @@ protected LSMVectorIndex(final DatabaseInternal database, final String name, fin
 
     // Create PQ file handler (for zero-disk-I/O search)
     // PQ data will be loaded after schema loads metadata (see loadVectorsAfterSchemaLoad)
-    this.pqFile = new LSMVectorIndexPQFile(filePath);
+    this.pqFile = createPQFileWithFallback(mutable.getFilePath());
 
     // Initialize compaction fields
     this.currentMutablePages = new AtomicInteger(mutable.getTotalPages());
@@ -309,6 +311,39 @@ protected LSMVectorIndex(final DatabaseInternal database, final String name, fin
     // See loadVectorsAfterSchemaLoad() method which is called by LSMVectorIndexMutable.onAfterSchemaLoad()
   }
 
+  private LSMVectorIndexPQFile createPQFileWithFallback(final String primaryBasePath) {
+    // Use the component file path as canonical. If legacy PQ exists at a shorter base name, migrate it once.
+    final LSMVectorIndexPQFile pq = new LSMVectorIndexPQFile(primaryBasePath);
+
+    // Derive a legacy base path by stripping the first extension (e.g., drop .4.262144.v0.lsmvecidx)
+    String legacyBasePath = null;
+    final int dot = primaryBasePath.indexOf('.');
+    if (dot > 0) {
+      legacyBasePath = primaryBasePath.substring(0, dot);
+    }
+
+    if (!pq.exists() && legacyBasePath != null) {
+      final LSMVectorIndexPQFile legacyPQ = new LSMVectorIndexPQFile(legacyBasePath);
+      if (legacyPQ.exists()) {
+        try {
+          final var targetParent = pq.getFilePath().getParent();
+          if (targetParent != null && !Files.exists(targetParent)) {
+            Files.createDirectories(targetParent);
+          }
+          Files.move(legacyPQ.getFilePath(), pq.getFilePath(), StandardCopyOption.REPLACE_EXISTING);
+          LogManager.instance().log(this, Level.INFO,
+                  "Migrated PQ file from legacy path %s to canonical %s", legacyPQ.getFilePath(), pq.getFilePath());
+        } catch (final Exception e) {
+          LogManager.instance().log(this, Level.WARNING,
+                  "Failed to migrate PQ file from legacy path %s to canonical %s: %s", legacyPQ.getFilePath(), pq.getFilePath(),
+                  e.getMessage());
+        }
+      }
+    }
+
+    return pq;
+  }
+
   /**
    * Load vectors from pages after schema has loaded metadata.
    * Called by LSMVectorIndexMutable.onAfterSchemaLoad() after dimensions are set from schema.json.