test(framework): add dual DB engine coverage with flaky test fixes (#6586)

* test: add dual DB engine (LevelDB + RocksDB) test coverage

Author: vividctrlalt

.github/workflows/pr-check.yml

@@ -178,6 +178,10 @@ jobs:
       - name: Build
         run: ./gradlew clean build --no-daemon
 
+      - name: Test with RocksDB engine
+        if: matrix.arch == 'x86_64'
+        run: ./gradlew :framework:testWithRocksDb --no-daemon
+
   docker-build-rockylinux:
     name: Build rockylinux (JDK 8 / x86_64)
     if: github.event.action != 'edited' && !failure()

common/src/main/java/org/tron/core/config/args/Storage.java

@@ -29,7 +29,6 @@
 import org.apache.commons.lang3.StringUtils;
 import org.iq80.leveldb.CompressionType;
 import org.iq80.leveldb.Options;
-import org.tron.common.arch.Arch;
 import org.tron.common.cache.CacheStrategies;
 import org.tron.common.cache.CacheType;
 import org.tron.common.utils.DbOptionalsUtils;
@@ -90,7 +89,6 @@ public class Storage {
    * Default values of directory
    */
   private static final String DEFAULT_DB_ENGINE = "LEVELDB";
-  private static final String ROCKS_DB_ENGINE = "ROCKSDB";
   private static final boolean DEFAULT_DB_SYNC = false;
   private static final boolean DEFAULT_EVENT_SUBSCRIBE_CONTRACT_PARSE = true;
   private static final String DEFAULT_DB_DIRECTORY = "database";
@@ -175,10 +173,6 @@ public class Storage {
   private final Map<String, Sha256Hash> dbRoots = Maps.newConcurrentMap();
 
   public static String getDbEngineFromConfig(final Config config) {
-    if (Arch.isArm64()) {
-      logger.warn("Arm64 architecture detected, using RocksDB as db engine, ignore config.");
-      return ROCKS_DB_ENGINE;
-    }
     return config.hasPath(DB_ENGINE_CONFIG_KEY)
         ? config.getString(DB_ENGINE_CONFIG_KEY) : DEFAULT_DB_ENGINE;
   }

framework/build.gradle

@@ -106,19 +106,38 @@ run {
     }
 }
 
-test {
-    retry {
+def configureTestTask = { Task t ->
+    t.retry {
         maxRetries = 5
         maxFailures = 20
     }
-    testLogging {
+    t.testLogging {
         exceptionFormat = 'full'
     }
+    if (isWindows()) {
+        t.exclude '**/ShieldedTransferActuatorTest.class'
+        t.exclude '**/BackupDbUtilTest.class'
+        t.exclude '**/ManagerTest.class'
+        t.exclude 'org/tron/core/zksnark/**'
+        t.exclude 'org/tron/common/runtime/vm/PrecompiledContractsVerifyProofTest.class'
+        t.exclude 'org/tron/core/ShieldedTRC20BuilderTest.class'
+        t.exclude 'org/tron/common/runtime/vm/WithdrawRewardTest.class'
+    }
+    t.maxHeapSize = "1024m"
+    t.doFirst {
+        t.forkEvery = 100
+        t.jvmArgs "-XX:MetaspaceSize=128m", "-XX:MaxMetaspaceSize=256m", "-XX:+UseG1GC"
+    }
+}
+
+test {
+    configureTestTask(it)
     jacoco {
         destinationFile = file("$buildDir/jacoco/jacocoTest.exec")
         classDumpDir = file("$buildDir/jacoco/classpathdumps")
     }
     if (rootProject.archInfo.isArm64) {
+        systemProperty 'storage.db.engine', 'ROCKSDB'
         exclude { element ->
             element.file.name.toLowerCase().contains('leveldb')
         }
@@ -129,21 +148,14 @@ test {
             excludeTestsMatching '*.*LevelDb*'
         }
     }
-    if (isWindows()) {
-        exclude '**/ShieldedTransferActuatorTest.class'
-        exclude '**/BackupDbUtilTest.class'
-        exclude '**/ManagerTest.class'
-        exclude 'org/tron/core/zksnark/**'
-        exclude 'org/tron/common/runtime/vm/PrecompiledContractsVerifyProofTest.class'
-        exclude 'org/tron/core/ShieldedTRC20BuilderTest.class'
-        exclude 'org/tron/common/runtime/vm/WithdrawRewardTest.class'
-    }
-    maxHeapSize = "1024m"
-    doFirst {
-        // Restart the JVM after every 100 tests to avoid memory leaks and ensure test isolation
-        forkEvery = 100
-        jvmArgs "-XX:MetaspaceSize=128m","-XX:MaxMetaspaceSize=256m", "-XX:+UseG1GC"
-    }
+}
+
+task testWithRocksDb(type: Test) {
+    description = 'Run tests with RocksDB engine'
+    group = 'verification'
+    configureTestTask(it)
+    systemProperty 'storage.db.engine', 'ROCKSDB'
+    exclude '**/LevelDbDataSourceImplTest.class'
 }
 
 jacocoTestReport {

framework/src/main/java/org/tron/common/logsfilter/nativequeue/NativeMessageQueue.java

@@ -51,10 +51,15 @@ public boolean start(int bindPort, int sendQueueLength) {
   public void stop() {
     if (Objects.nonNull(publisher)) {
       publisher.close();
+      publisher = null;
     }
 
     if (Objects.nonNull(context)) {
       context.close();
+      context = null;
+    }
+    synchronized (NativeMessageQueue.class) {
+      instance = null;
     }
   }
 

framework/src/main/java/org/tron/core/config/args/Args.java

@@ -130,10 +130,27 @@ public static void setParam(final String[] args, final String confFileName) {
     // 3. CLI overrides Config (highest priority)
     applyCLIParams(cmd, jc);
 
-    // 4. Init witness (depends on CLI witness flag)
+    // 4. Apply platform constraints (e.g. ARM64 forces RocksDB)
+    applyPlatformConstraints();
+
+    // 5. Init witness (depends on CLI witness flag)
     initLocalWitnesses(config, cmd);
   }
 
+  /**
+   * Apply platform-specific constraints after all config sources are resolved.
+   * ARM64 does not support LevelDB (native JNI library unavailable),
+   * so db.engine is forced to RocksDB regardless of config or CLI settings.
+   */
+  private static void applyPlatformConstraints() {
+    if (Arch.isArm64()
+        && !Constant.ROCKSDB.equalsIgnoreCase(PARAMETER.storage.getDbEngine())) {
+      logger.warn("ARM64 only supports RocksDB, ignoring db.engine='{}'",
+          PARAMETER.storage.getDbEngine());
+      PARAMETER.storage.setDbEngine(Constant.ROCKSDB);
+    }
+  }
+
   /**
    * Apply parameters from config file.
    */
@@ -784,11 +801,8 @@ public static void applyConfigParams(
             .getInt(ConfigKey.COMMITTEE_ALLOW_OPTIMIZED_RETURN_VALUE_OF_CHAIN_ID) : 0;
 
     initBackupProperty(config);
-    if (Constant.ROCKSDB.equalsIgnoreCase(CommonParameter
-        .getInstance().getStorage().getDbEngine())) {
-      initRocksDbBackupProperty(config);
-      initRocksDbSettings(config);
-    }
+    initRocksDbBackupProperty(config);
+    initRocksDbSettings(config);
 
     PARAMETER.actuatorSet =
         config.hasPath(ConfigKey.ACTUATOR_WHITELIST)

framework/src/main/java/org/tron/core/db/backup/NeedBeanCondition.java

@@ -9,7 +9,12 @@ public class NeedBeanCondition implements Condition {
 
   @Override
   public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
-    return ("ROCKSDB".equals(Args.getInstance().getStorage().getDbEngine().toUpperCase()))
+    if (Args.getInstance() == null || Args.getInstance().getStorage() == null
+        || Args.getInstance().getStorage().getDbEngine() == null
+        || Args.getInstance().getDbBackupConfig() == null) {
+      return false;
+    }
+    return "ROCKSDB".equalsIgnoreCase(Args.getInstance().getStorage().getDbEngine())
         && Args.getInstance().getDbBackupConfig().isEnable() && !Args.getInstance().isWitness();
   }
 }

framework/src/test/java/org/tron/common/BaseMethodTest.java

@@ -0,0 +1,95 @@
+package org.tron.common;
+
+import java.io.IOException;
+import java.util.Arrays;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+import org.tron.common.application.Application;
+import org.tron.common.application.ApplicationFactory;
+import org.tron.common.application.TronApplicationContext;
+import org.tron.core.ChainBaseManager;
+import org.tron.core.config.DefaultConfig;
+import org.tron.core.config.args.Args;
+import org.tron.core.db.Manager;
+
+/**
+ * Base class for tests that need a fresh Spring context per test method.
+ *
+ * Each @Test method gets its own TronApplicationContext, created in @Before
+ * and destroyed in @After, ensuring full isolation between tests.
+ *
+ * Subclasses can customize behavior by overriding hook methods:
+ *   extraArgs()      — additional CLI args (e.g. "--debug")
+ *   configFile()     — config file (default: config-test.conf)
+ *   beforeContext()  — runs after Args.setParam, before Spring context creation
+ *   afterInit()      — runs after Spring context is ready (e.g. get extra beans)
+ *   beforeDestroy()  — runs before context shutdown (e.g. close resources)
+ *
+ * Use this when:
+ *   - methods modify database state that would affect other methods
+ *   - methods need different CommonParameter settings (e.g. setP2pDisable)
+ *   - methods need beforeContext() to configure state before Spring starts
+ *
+ * If methods are read-only and don't interfere with each other, use BaseTest instead.
+ * Tests that don't need Spring (e.g. pure unit tests) should NOT extend either base class.
+ */
+@Slf4j
+public abstract class BaseMethodTest {
+
+  @Rule
+  public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+  protected TronApplicationContext context;
+  protected Application appT;
+  protected Manager dbManager;
+  protected ChainBaseManager chainBaseManager;
+
+  protected String[] extraArgs() {
+    return new String[0];
+  }
+
+  protected String configFile() {
+    return TestConstants.TEST_CONF;
+  }
+
+  @Before
+  public final void initContext() throws IOException {
+    String[] baseArgs = new String[]{
+        "--output-directory", temporaryFolder.newFolder().toString()};
+    String[] allArgs = mergeArgs(baseArgs, extraArgs());
+    Args.setParam(allArgs, configFile());
+    beforeContext();
+    context = new TronApplicationContext(DefaultConfig.class);
+    appT = ApplicationFactory.create(context);
+    dbManager = context.getBean(Manager.class);
+    chainBaseManager = context.getBean(ChainBaseManager.class);
+    afterInit();
+  }
+
+  protected void beforeContext() {
+  }
+
+  protected void afterInit() {
+  }
+
+  @After
+  public final void destroyContext() {
+    beforeDestroy();
+    if (context != null) {
+      context.close(); // triggers appT.shutdown() via TronApplicationContext
+    }
+    Args.clearParam();
+  }
+
+  protected void beforeDestroy() {
+  }
+
+  private static String[] mergeArgs(String[] base, String[] extra) {
+    String[] result = Arrays.copyOf(base, base.length + extra.length);
+    System.arraycopy(extra, 0, result, base.length, extra.length);
+    return result;
+  }
+}

framework/src/test/java/org/tron/common/BaseTest.java

@@ -30,6 +30,31 @@
 import org.tron.core.store.AccountStore;
 import org.tron.protos.Protocol;
 
+/**
+ * Base class for tests that need a Spring context (Manager, ChainBaseManager, etc.).
+ * The context is created once per test class.
+ *
+ * Args.setParam() is called in a static block, the context is shared by all @Test
+ * methods in the class, and destroyed in @AfterClass. This is faster but means
+ * test methods within the same class are not isolated from each other.
+ *
+ * Use this when:
+ *   - test methods are read-only or don't interfere with each other
+ *   - no need to change CommonParameter/Args between methods
+ *
+ * Use BaseMethodTest instead when:
+ *   - methods modify database state that would affect other methods
+ *   - methods need different CommonParameter settings (e.g. setP2pDisable)
+ *   - methods need beforeContext() to configure state before Spring starts
+ *
+ * Tests that don't need Spring (e.g. pure unit tests like ArgsTest,
+ * LevelDbDataSourceImplTest) should NOT extend either base class.
+ *
+ * Subclasses must call Args.setParam() in a static initializer block:
+ *   static {
+ *     Args.setParam(new String[]{"--output-directory", dbPath()}, TEST_CONF);
+ *   }
+ */
 @Slf4j
 @RunWith(SpringJUnit4ClassRunner.class)
 @ContextConfiguration(classes = {DefaultConfig.class})

framework/src/test/java/org/tron/common/TestConstants.java

@@ -1,7 +1,39 @@
 package org.tron.common;
 
+import static org.junit.Assume.assumeFalse;
+
+import org.tron.common.arch.Arch;
+
+/**
+ * Centralized test environment constants and utilities.
+ *
+ * <h3>DB engine override for dual-engine testing</h3>
+ * Gradle tasks ({@code test} on ARM64, {@code testWithRocksDb}) set the JVM system property
+ * {@code -Dstorage.db.engine=ROCKSDB}. Because test config files are loaded from the classpath
+ * via {@code ConfigFactory.load(fileName)}, Typesafe Config automatically merges system
+ * properties with higher priority than the config file values. This means the config's
+ * {@code storage.db.engine = "LEVELDB"} is overridden transparently, without any code changes
+ * in individual tests.
+ *
+ * <p><b>IMPORTANT:</b> Config files MUST be classpath resources (in {@code src/test/resources/}),
+ * NOT standalone files in the working directory. If a config file exists on disk,
+ * {@code Configuration.getByFileName} falls back to {@code ConfigFactory.parseFile()},
+ * which does NOT merge system properties, and the engine override will silently fail.
+ */
 public class TestConstants {
 
   public static final String TEST_CONF = "config-test.conf";
   public static final String NET_CONF = "config.conf";
+  public static final String MAINNET_CONF = "config-test-mainnet.conf";
+  public static final String DBBACKUP_CONF = "config-test-dbbackup.conf";
+  public static final String LOCAL_CONF = "config-localtest.conf";
+  public static final String STORAGE_CONF = "config-test-storagetest.conf";
+  public static final String INDEX_CONF = "config-test-index.conf";
+
+  /**
+   * Skips the current test on ARM64 where LevelDB JNI is unavailable.
+   */
+  public static void assumeLevelDbAvailable() {
+    assumeFalse("LevelDB JNI unavailable on ARM64", Arch.isArm64());
+  }
 }