Skip to content

Commit 8775137

Browse files
halibobo1205halibobo1205
committed
feat(rocksdb): release option when db is created
1 parent 6e8742a commit 8775137

11 files changed

Lines changed: 59 additions & 219 deletions

File tree

‎chainbase/src/main/java/org/tron/common/storage/OptionsPicker.java‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java‎

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,10 @@
4141
import lombok.extern.slf4j.Slf4j;
4242
import org.iq80.leveldb.DB;
4343
import org.iq80.leveldb.DBIterator;
44-
import org.iq80.leveldb.Logger;
4544
import org.iq80.leveldb.Options;
4645
import org.iq80.leveldb.ReadOptions;
4746
import org.iq80.leveldb.WriteBatch;
4847
import org.iq80.leveldb.WriteOptions;
49-
import org.slf4j.LoggerFactory;
5048
import org.tron.common.parameter.CommonParameter;
5149
import org.tron.common.storage.WriteOptionsWrapper;
5250
import org.tron.common.storage.metric.DbStat;
@@ -70,28 +68,11 @@ public class LevelDbDataSourceImpl extends DbStat implements DbSourceInter<byte[
7068
private Options options;
7169
private WriteOptions writeOptions;
7270
private ReadWriteLock resetDbLock = new ReentrantReadWriteLock();
73-
private static final org.slf4j.Logger innerLogger = LoggerFactory.getLogger(LEVELDB);
74-
private Logger leveldbLogger = new Logger() {
75-
@Override
76-
public void log(String message) {
77-
innerLogger.info("{} {}", dataBaseName, message);
78-
}
79-
};
71+
8072

8173
/**
8274
* constructor.
8375
*/
84-
public LevelDbDataSourceImpl(String parentPath, String dataBaseName, Options options,
85-
WriteOptions writeOptions) {
86-
this.parentPath = Paths.get(
87-
parentPath,
88-
CommonParameter.getInstance().getStorage().getDbDirectory()
89-
).toString();
90-
this.dataBaseName = dataBaseName;
91-
this.options = options.logger(leveldbLogger);
92-
this.writeOptions = writeOptions;
93-
initDB();
94-
}
9576

9677
public LevelDbDataSourceImpl(String parentPath, String dataBaseName) {
9778
this.parentPath = Paths.get(
@@ -100,8 +81,10 @@ public LevelDbDataSourceImpl(String parentPath, String dataBaseName) {
10081
).toString();
10182

10283
this.dataBaseName = dataBaseName;
103-
options = new Options().logger(leveldbLogger);
104-
writeOptions = new WriteOptions();
84+
this.options = StorageUtils.getOptionsByDbName(dataBaseName);
85+
this.writeOptions = new WriteOptions().sync(CommonParameter.getInstance()
86+
.getStorage().isDbSync());
87+
initDB();
10588
}
10689

10790
@Override
@@ -455,7 +438,7 @@ public Stream<Entry<byte[], byte[]>> stream() {
455438
@Override
456439
public LevelDbDataSourceImpl newInstance() {
457440
return new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(dataBaseName),
458-
dataBaseName, options, writeOptions);
441+
dataBaseName);
459442
}
460443

461444
private DBIterator getDBIterator() {

‎chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java‎

Lines changed: 8 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@
2222
import lombok.NoArgsConstructor;
2323
import lombok.extern.slf4j.Slf4j;
2424
import org.rocksdb.Checkpoint;
25-
import org.rocksdb.InfoLogLevel;
26-
import org.rocksdb.Logger;
2725
import org.rocksdb.Options;
2826
import org.rocksdb.ReadOptions;
2927
import org.rocksdb.RocksDB;
@@ -32,7 +30,6 @@
3230
import org.rocksdb.Status;
3331
import org.rocksdb.WriteBatch;
3432
import org.rocksdb.WriteOptions;
35-
import org.slf4j.LoggerFactory;
3633
import org.tron.common.error.TronDBException;
3734
import org.tron.common.setting.RocksDbSettings;
3835
import org.tron.common.storage.WriteOptionsWrapper;
@@ -52,26 +49,16 @@ public class RocksDbDataSourceImpl extends DbStat implements DbSourceInter<byte[
5249

5350
private String dataBaseName;
5451
private RocksDB database;
55-
private Options options;
5652
private volatile boolean alive;
5753
private String parentPath;
5854
private ReadWriteLock resetDbLock = new ReentrantReadWriteLock();
59-
private static final org.slf4j.Logger rocksDbLogger = LoggerFactory.getLogger(ROCKSDB);
6055

61-
public RocksDbDataSourceImpl(String parentPath, String name, Options options) {
56+
public RocksDbDataSourceImpl(String parentPath, String name) {
6257
this.dataBaseName = name;
6358
this.parentPath = parentPath;
64-
this.options = options;
6559
initDB();
6660
}
6761

68-
@VisibleForTesting
69-
public RocksDbDataSourceImpl(String parentPath, String name) {
70-
this.parentPath = parentPath;
71-
this.dataBaseName = name;
72-
this.options = RocksDbSettings.getOptionsByDbName(name);
73-
}
74-
7562
public Path getDbPath() {
7663
return Paths.get(parentPath, dataBaseName);
7764
}
@@ -192,14 +179,8 @@ public void initDB() {
192179
if (dataBaseName == null) {
193180
throw new IllegalArgumentException("No name set to the dbStore");
194181
}
195-
options.setLogger(new Logger(options) {
196-
@Override
197-
protected void log(InfoLogLevel infoLogLevel, String logMsg) {
198-
rocksDbLogger.info("{} {}", dataBaseName, logMsg);
199-
}
200-
});
201182

202-
try {
183+
try (Options options = RocksDbSettings.getOptionsByDbName(dataBaseName)) {
203184
logger.debug("Opening database {}.", dataBaseName);
204185
final Path dbPath = getDbPath();
205186

@@ -415,14 +396,15 @@ public Set<byte[]> getValuesNext(byte[] key, long limit) {
415396

416397
public void backup(String dir) throws RocksDBException {
417398
throwIfNotAlive();
418-
Checkpoint cp = Checkpoint.create(database);
419-
cp.createCheckpoint(dir + this.getDBName());
399+
try (Checkpoint cp = Checkpoint.create(database)) {
400+
cp.createCheckpoint(dir + this.getDBName());
401+
}
420402
}
421403

422404
private RocksIterator getRocksIterator() {
423-
try ( ReadOptions readOptions = new ReadOptions().setFillCache(false)) {
405+
try (ReadOptions readOptions = new ReadOptions().setFillCache(false)) {
424406
throwIfNotAlive();
425-
return database.newIterator(readOptions);
407+
return database.newIterator(readOptions);
426408
}
427409
}
428410

@@ -432,8 +414,7 @@ public boolean deleteDbBakPath(String dir) {
432414

433415
@Override
434416
public RocksDbDataSourceImpl newInstance() {
435-
return new RocksDbDataSourceImpl(parentPath, dataBaseName,
436-
this.options);
417+
return new RocksDbDataSourceImpl(parentPath, dataBaseName);
437418
}
438419

439420

‎chainbase/src/main/java/org/tron/common/utils/StorageUtils.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
package org.tron.common.utils;
22

33
import static org.tron.common.parameter.CommonParameter.ENERGY_LIMIT_HARD_FORK;
4+
import static org.tron.core.db.common.DbSourceInter.LEVELDB;
45

56
import java.io.File;
67
import org.apache.commons.lang3.StringUtils;
78
import org.iq80.leveldb.Options;
9+
import org.slf4j.LoggerFactory;
810
import org.tron.common.parameter.CommonParameter;
911
import org.tron.core.Constant;
1012

1113

1214
public class StorageUtils {
1315

16+
private static final org.slf4j.Logger levelDbLogger = LoggerFactory.getLogger(LEVELDB);
17+
1418
public static boolean getEnergyLimitHardFork() {
1519
return ENERGY_LIMIT_HARD_FORK;
1620
}
@@ -62,6 +66,7 @@ public static Options getOptionsByDbName(String dbName) {
6266
if (Constant.MARKET_PAIR_PRICE_TO_ORDER.equals(dbName)) {
6367
options.comparator(new MarketOrderPriceComparatorForLevelDB());
6468
}
69+
options.logger(message -> levelDbLogger.info("{} {}", dbName, message));
6570
return options;
6671
}
6772
}

‎chainbase/src/main/java/org/tron/core/db/TronDatabase.java‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,8 @@
88
import javax.annotation.PostConstruct;
99
import lombok.Getter;
1010
import lombok.extern.slf4j.Slf4j;
11-
import org.iq80.leveldb.WriteOptions;
1211
import org.springframework.beans.factory.annotation.Autowired;
1312
import org.tron.common.parameter.CommonParameter;
14-
import org.tron.common.storage.OptionsPicker;
1513
import org.tron.common.storage.WriteOptionsWrapper;
1614
import org.tron.common.storage.leveldb.LevelDbDataSourceImpl;
1715
import org.tron.common.storage.metric.DbStatService;
@@ -24,7 +22,7 @@
2422
import org.tron.core.exception.ItemNotFoundException;
2523

2624
@Slf4j(topic = "DB")
27-
public abstract class TronDatabase<T> extends OptionsPicker implements ITronChainBase<T> {
25+
public abstract class TronDatabase<T> implements ITronChainBase<T> {
2826

2927
protected DbSourceInter<byte[]> dbSource;
3028
@Getter
@@ -40,21 +38,13 @@ protected TronDatabase(String dbName) {
4038

4139
if ("LEVELDB".equals(CommonParameter.getInstance().getStorage()
4240
.getDbEngine().toUpperCase())) {
43-
dbSource =
44-
new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(dbName),
45-
dbName,
46-
getOptionsByDbNameForLevelDB(dbName),
47-
new WriteOptions().sync(CommonParameter.getInstance()
48-
.getStorage().isDbSync()));
41+
dbSource = new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(dbName), dbName);
4942
} else if ("ROCKSDB".equals(CommonParameter.getInstance()
5043
.getStorage().getDbEngine().toUpperCase())) {
5144
String parentName = Paths.get(StorageUtils.getOutputDirectoryByDbName(dbName),
5245
CommonParameter.getInstance().getStorage().getDbDirectory()).toString();
53-
dbSource =
54-
new RocksDbDataSourceImpl(parentName, dbName, getOptionsByDbNameForRocksDB(dbName));
46+
dbSource = new RocksDbDataSourceImpl(parentName, dbName);
5547
}
56-
57-
dbSource.initDB();
5848
}
5949

6050
@PostConstruct

‎chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,8 @@
1515
import javax.annotation.PostConstruct;
1616
import lombok.Getter;
1717
import lombok.extern.slf4j.Slf4j;
18-
import org.iq80.leveldb.WriteOptions;
1918
import org.springframework.beans.factory.annotation.Autowired;
2019
import org.tron.common.parameter.CommonParameter;
21-
import org.tron.common.storage.OptionsPicker;
2220
import org.tron.common.storage.leveldb.LevelDbDataSourceImpl;
2321
import org.tron.common.storage.metric.DbStatService;
2422
import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl;
@@ -38,7 +36,7 @@
3836

3937

4038
@Slf4j(topic = "DB")
41-
public abstract class TronStoreWithRevoking<T extends ProtoCapsule> extends OptionsPicker implements ITronChainBase<T> {
39+
public abstract class TronStoreWithRevoking<T extends ProtoCapsule> implements ITronChainBase<T> {
4240

4341
@Getter // only for unit test
4442
protected IRevokingDB revokingDB;
@@ -58,18 +56,12 @@ protected TronStoreWithRevoking(String dbName) {
5856
String dbEngine = CommonParameter.getInstance().getStorage().getDbEngine();
5957
if ("LEVELDB".equals(dbEngine.toUpperCase())) {
6058
this.db = new LevelDB(
61-
new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(dbName),
62-
dbName,
63-
getOptionsByDbNameForLevelDB(dbName),
64-
new WriteOptions().sync(CommonParameter.getInstance()
65-
.getStorage().isDbSync())));
59+
new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(dbName), dbName));
6660
} else if ("ROCKSDB".equals(dbEngine.toUpperCase())) {
6761
String parentPath = Paths
6862
.get(StorageUtils.getOutputDirectoryByDbName(dbName), CommonParameter
6963
.getInstance().getStorage().getDbDirectory()).toString();
70-
this.db = new RocksDB(
71-
new RocksDbDataSourceImpl(parentPath,
72-
dbName, getOptionsByDbNameForRocksDB(dbName)));
64+
this.db = new RocksDB(new RocksDbDataSourceImpl(parentPath, dbName));
7365
} else {
7466
throw new RuntimeException(String.format("db engine %s is error", dbEngine));
7567
}

‎chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,9 @@
3131
import lombok.extern.slf4j.Slf4j;
3232
import org.apache.commons.lang3.ArrayUtils;
3333
import org.bouncycastle.util.encoders.Hex;
34-
import org.iq80.leveldb.WriteOptions;
3534
import org.tron.common.parameter.CommonParameter;
3635
import org.tron.common.prometheus.MetricKeys;
3736
import org.tron.common.prometheus.Metrics;
38-
import org.tron.common.storage.OptionsPicker;
3937
import org.tron.common.storage.leveldb.LevelDbDataSourceImpl;
4038
import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl;
4139
import org.tron.common.utils.ByteArray;
@@ -49,7 +47,7 @@
4947
import org.tron.core.store.DynamicPropertiesStore;
5048

5149
@Slf4j(topic = "DB")
52-
public class TxCacheDB extends OptionsPicker implements DB<byte[], byte[]>, Flusher {
50+
public class TxCacheDB implements DB<byte[], byte[]>, Flusher {
5351

5452
// > 65_536(= 2^16) blocks, that is the number of the reference block
5553
private static final long MAX_BLOCK_SIZE = 65536;
@@ -106,17 +104,13 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore,
106104
String dbEngine = CommonParameter.getInstance().getStorage().getDbEngine();
107105
if ("LEVELDB".equals(dbEngine.toUpperCase())) {
108106
this.persistentStore = new LevelDB(
109-
new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(name),
110-
name, getOptionsByDbNameForLevelDB(name),
111-
new WriteOptions().sync(CommonParameter.getInstance()
112-
.getStorage().isDbSync())));
107+
new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(name), name));
113108
} else if ("ROCKSDB".equals(dbEngine.toUpperCase())) {
114109
String parentPath = Paths
115110
.get(StorageUtils.getOutputDirectoryByDbName(name), CommonParameter
116111
.getInstance().getStorage().getDbDirectory()).toString();
117112

118-
this.persistentStore = new RocksDB(
119-
new RocksDbDataSourceImpl(parentPath, name, getOptionsByDbNameForRocksDB(name)));
113+
this.persistentStore = new RocksDB(new RocksDbDataSourceImpl(parentPath, name));
120114
} else {
121115
throw new RuntimeException(String.format("db type: %s is not supported", dbEngine));
122116
}

‎common/src/main/java/org/tron/common/setting/RocksDbSettings.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,20 @@
11
package org.tron.common.setting;
22

3+
import static org.tron.core.Constant.ROCKSDB;
4+
35
import java.util.Arrays;
46
import lombok.Getter;
57
import lombok.extern.slf4j.Slf4j;
68
import org.rocksdb.BlockBasedTableConfig;
79
import org.rocksdb.BloomFilter;
810
import org.rocksdb.ComparatorOptions;
11+
import org.rocksdb.InfoLogLevel;
912
import org.rocksdb.LRUCache;
13+
import org.rocksdb.Logger;
1014
import org.rocksdb.Options;
1115
import org.rocksdb.RocksDB;
1216
import org.rocksdb.Statistics;
17+
import org.slf4j.LoggerFactory;
1318
import org.tron.common.utils.MarketOrderPriceComparatorForRocksDB;
1419
import org.tron.core.Constant;
1520

@@ -54,6 +59,8 @@ public class RocksDbSettings {
5459
"GITLAB_CI"
5560
};
5661

62+
private static final org.slf4j.Logger rocksDbLogger = LoggerFactory.getLogger(ROCKSDB);
63+
5764
private RocksDbSettings() {
5865

5966
}
@@ -161,6 +168,12 @@ public static Options getOptionsByDbName(String dbName) {
161168

162169
Options options = new Options();
163170

171+
options.setLogger(new Logger(options) {
172+
@Override
173+
protected void log(InfoLogLevel infoLogLevel, String logMsg) {
174+
rocksDbLogger.info("{} {}", dbName, logMsg);
175+
}
176+
});
164177
// most of these options are suggested by https://github.com/facebook/rocksdb/wiki/Set-Up-Options
165178

166179
// general options
@@ -196,6 +209,7 @@ public static Options getOptionsByDbName(String dbName) {
196209
}
197210

198211
if (isRunningInCI()) {
212+
options.optimizeForSmallDb();
199213
// Disable fallocate calls to avoid issues with disk space
200214
options.setAllowFAllocate(false);
201215
// Set WAL size limits to avoid excessive disk

0 commit comments

Comments
 (0)