diskquota/jdbc: poll on FK migration race recovery

Concurrent migrateForeignKeys callers could fail OracleForeignKey
MigrationIT.migrateIsConcurrentStartupSafe sporadically. Oracle's
ALTER TABLE uses NOWAIT, and a peer thread that gets ORA-00054 on
DROP CONSTRAINT could re-check the FK state while the peer was still
between its drop and its add, observe an in-between state, and
rethrow.

on-behalf-of: @camptocamp <info@camptocamp.com>

Author: groldan

geowebcache/diskquota/jdbc/src/main/java/org/geowebcache/diskquota/jdbc/SQLDialect.java

@@ -184,7 +184,7 @@ private Void upgradeTilepageForeignKey(DatabaseMetaData dbmd, String schema, Sim
                     jdbcOperations.execute(drop);
                     jdbcOperations.execute(add);
                 } catch (DataAccessException raceLikely) {
-                    if (isTilepageFkAlreadyMigrated(dbmd, schema, tilepageName)) {
+                    if (awaitConcurrentMigration(dbmd, schema, tilepageName)) {
                         LOG.fine(() -> "TILEPAGE FK was migrated concurrently by another instance "
                                 + "while this instance was trying to drop %s; accepting concurrent migration"
                                         .formatted(fkName));
@@ -212,6 +212,31 @@ protected String tilepageFkAddSql(String prefixedTilepageName, String prefix) {
                 .formatted(prefixedTilepageName, prefix);
     }
 
+    private static final int CONCURRENT_MIGRATION_RECHECK_ATTEMPTS = 20;
+    private static final long CONCURRENT_MIGRATION_RECHECK_BACKOFF_MS = 100L;
+
+    /**
+     * Polls {@link #isTilepageFkAlreadyMigrated} after a failed drop/add to absorb the window in which a peer instance
+     * is mid-migration. Oracle's {@code ALTER TABLE} uses NOWAIT, so a concurrent peer can fail us with ORA-00054 while
+     * it still has the drop committed but not yet the add; a one-shot recheck would observe the in-flight state and
+     * give up.
+     */
+    private boolean awaitConcurrentMigration(DatabaseMetaData dbmd, String schema, String tilepageName)
+            throws SQLException {
+        for (int attempt = 0; attempt < CONCURRENT_MIGRATION_RECHECK_ATTEMPTS; attempt++) {
+            if (isTilepageFkAlreadyMigrated(dbmd, schema, tilepageName)) {
+                return true;
+            }
+            try {
+                Thread.sleep(CONCURRENT_MIGRATION_RECHECK_BACKOFF_MS);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                return false;
+            }
+        }
+        return false;
+    }
+
     /** Re-checks FK state after a failed migration attempt to detect a concurrent migration from another instance. */
     private boolean isTilepageFkAlreadyMigrated(DatabaseMetaData dbmd, String schema, String tilepageName)
             throws SQLException {