Skip to content

Commit c9afbb3

Browse files
halibobo1205halibobo1205
committed
feat(db): update leveldb and rocksdb for arm
1 parent 04e4358 commit c9afbb3

36 files changed

Lines changed: 540 additions & 432 deletions

File tree

‎build.gradle‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ subprojects {
100100
reproducibleFileOrder = true
101101
duplicatesStrategy = DuplicatesStrategy.INCLUDE // allow duplicates
102102
}
103+
tasks.withType(Test).configureEach {
104+
// https://docs.gradle.org/current/dsl/org.gradle.api.tasks.testing.Test.html#org.gradle.api.tasks.testing.Test:environment
105+
environment 'CI', 'true'
106+
}
103107
}
104108

105109
task copyToParent(type: Copy) {

‎chainbase/src/main/java/org/tron/common/storage/OptionsPicker.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package org.tron.common.storage;
2+
3+
import org.tron.common.setting.RocksDbSettings;
4+
import org.tron.common.utils.StorageUtils;
5+
6+
public class OptionsPicker {
7+
8+
protected org.iq80.leveldb.Options getOptionsByDbNameForLevelDB(String dbName) {
9+
return StorageUtils.getOptionsByDbName(dbName);
10+
}
11+
12+
protected org.rocksdb.Options getOptionsByDbNameForRocksDB(String dbName) {
13+
return RocksDbSettings.getOptionsByDbName(dbName);
14+
}
15+
}

‎chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java‎

Lines changed: 32 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,14 @@
2222
import java.util.stream.Collectors;
2323
import lombok.NoArgsConstructor;
2424
import lombok.extern.slf4j.Slf4j;
25-
import org.rocksdb.BlockBasedTableConfig;
26-
import org.rocksdb.BloomFilter;
2725
import org.rocksdb.Checkpoint;
28-
import org.rocksdb.DirectComparator;
2926
import org.rocksdb.InfoLogLevel;
3027
import org.rocksdb.Logger;
3128
import org.rocksdb.Options;
3229
import org.rocksdb.ReadOptions;
3330
import org.rocksdb.RocksDB;
3431
import org.rocksdb.RocksDBException;
3532
import org.rocksdb.RocksIterator;
36-
import org.rocksdb.Statistics;
3733
import org.rocksdb.Status;
3834
import org.rocksdb.WriteBatch;
3935
import org.rocksdb.WriteOptions;
@@ -56,36 +52,28 @@
5652
public class RocksDbDataSourceImpl extends DbStat implements DbSourceInter<byte[]>,
5753
Iterable<Map.Entry<byte[], byte[]>>, Instance<RocksDbDataSourceImpl> {
5854

59-
ReadOptions readOpts;
6055
private String dataBaseName;
6156
private RocksDB database;
57+
private Options options;
6258
private volatile boolean alive;
6359
private String parentPath;
6460
private ReadWriteLock resetDbLock = new ReentrantReadWriteLock();
6561
private static final String KEY_ENGINE = "ENGINE";
6662
private static final String ROCKSDB = "ROCKSDB";
67-
private DirectComparator comparator;
6863
private static final org.slf4j.Logger rocksDbLogger = LoggerFactory.getLogger(ROCKSDB);
6964

70-
public RocksDbDataSourceImpl(String parentPath, String name, RocksDbSettings settings,
71-
DirectComparator comparator) {
65+
public RocksDbDataSourceImpl(String parentPath, String name, Options options) {
7266
this.dataBaseName = name;
7367
this.parentPath = parentPath;
74-
this.comparator = comparator;
75-
RocksDbSettings.setRocksDbSettings(settings);
76-
initDB();
77-
}
78-
79-
public RocksDbDataSourceImpl(String parentPath, String name, RocksDbSettings settings) {
80-
this.dataBaseName = name;
81-
this.parentPath = parentPath;
82-
RocksDbSettings.setRocksDbSettings(settings);
68+
this.options = options;
8369
initDB();
8470
}
8571

72+
@VisibleForTesting
8673
public RocksDbDataSourceImpl(String parentPath, String name) {
8774
this.parentPath = parentPath;
8875
this.dataBaseName = name;
76+
this.options = RocksDbSettings.getOptionsByDbName(name);
8977
}
9078

9179
public Path getDbPath() {
@@ -232,10 +220,6 @@ public void initDB() {
232220
"Cannot open LevelDB database '%s' with RocksDB engine."
233221
+ " Set db.engine=LEVELDB or use RocksDB database. ", dataBaseName));
234222
}
235-
initDB(RocksDbSettings.getSettings());
236-
}
237-
238-
public void initDB(RocksDbSettings settings) {
239223
resetDbLock.writeLock().lock();
240224
try {
241225
if (isAlive()) {
@@ -244,81 +228,40 @@ public void initDB(RocksDbSettings settings) {
244228
if (dataBaseName == null) {
245229
throw new IllegalArgumentException("No name set to the dbStore");
246230
}
231+
options.setLogger(new Logger(options) {
232+
@Override
233+
protected void log(InfoLogLevel infoLogLevel, String logMsg) {
234+
rocksDbLogger.info("{} {}", dataBaseName, logMsg);
235+
}
236+
});
247237

248-
try (Options options = new Options()) {
249-
250-
// most of these options are suggested by https://github.com/facebook/rocksdb/wiki/Set-Up-Options
238+
try {
239+
logger.debug("Opening database {}.", dataBaseName);
240+
final Path dbPath = getDbPath();
251241

252-
// general options
253-
if (settings.isEnableStatistics()) {
254-
options.setStatistics(new Statistics());
255-
options.setStatsDumpPeriodSec(60);
256-
}
257-
options.setCreateIfMissing(true);
258-
options.setIncreaseParallelism(1);
259-
options.setLevelCompactionDynamicLevelBytes(true);
260-
options.setMaxOpenFiles(settings.getMaxOpenFiles());
261-
262-
// general options supported user config
263-
options.setNumLevels(settings.getLevelNumber());
264-
options.setMaxBytesForLevelMultiplier(settings.getMaxBytesForLevelMultiplier());
265-
options.setMaxBytesForLevelBase(settings.getMaxBytesForLevelBase());
266-
options.setMaxBackgroundCompactions(settings.getCompactThreads());
267-
options.setLevel0FileNumCompactionTrigger(settings.getLevel0FileNumCompactionTrigger());
268-
options.setTargetFileSizeMultiplier(settings.getTargetFileSizeMultiplier());
269-
options.setTargetFileSizeBase(settings.getTargetFileSizeBase());
270-
if (comparator != null) {
271-
options.setComparator(comparator);
242+
if (!Files.isSymbolicLink(dbPath.getParent())) {
243+
Files.createDirectories(dbPath.getParent());
272244
}
273-
options.setLogger(new Logger(options) {
274-
@Override
275-
protected void log(InfoLogLevel infoLogLevel, String logMsg) {
276-
rocksDbLogger.info("{} {}", dataBaseName, logMsg);
277-
}
278-
});
279-
280-
// table options
281-
final BlockBasedTableConfig tableCfg;
282-
options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig());
283-
tableCfg.setBlockSize(settings.getBlockSize());
284-
tableCfg.setBlockCache(RocksDbSettings.getCache());
285-
tableCfg.setCacheIndexAndFilterBlocks(true);
286-
tableCfg.setPinL0FilterAndIndexBlocksInCache(true);
287-
tableCfg.setFilter(new BloomFilter(10, false));
288-
289-
// read options
290-
readOpts = new ReadOptions();
291-
readOpts = readOpts.setPrefixSameAsStart(true)
292-
.setVerifyChecksums(false);
293245

294246
try {
295-
logger.debug("Opening database {}.", dataBaseName);
296-
final Path dbPath = getDbPath();
297-
298-
if (!Files.isSymbolicLink(dbPath.getParent())) {
299-
Files.createDirectories(dbPath.getParent());
247+
database = RocksDB.open(options, dbPath.toString());
248+
} catch (RocksDBException e) {
249+
if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) {
250+
logger.error("Database {} corrupted, please delete database directory({}) "
251+
+ "and restart.", dataBaseName, parentPath, e);
252+
} else {
253+
logger.error("Open Database {} failed", dataBaseName, e);
300254
}
301-
302-
try {
303-
database = RocksDB.open(options, dbPath.toString());
304-
} catch (RocksDBException e) {
305-
if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) {
306-
logger.error("Database {} corrupted, please delete database directory({}) " +
307-
"and restart.", dataBaseName, parentPath, e);
308-
} else {
309-
logger.error("Open Database {} failed", dataBaseName, e);
310-
}
311-
throw new TronError(e, TronError.ErrCode.ROCKSDB_INIT);
312-
}
313-
314-
alive = true;
315-
} catch (IOException ioe) {
316-
throw new RuntimeException(
317-
String.format("failed to init database: %s", dataBaseName), ioe);
255+
throw new TronError(e, TronError.ErrCode.ROCKSDB_INIT);
318256
}
319257

320-
logger.debug("Init DB {} done.", dataBaseName);
258+
alive = true;
259+
} catch (IOException ioe) {
260+
throw new RuntimeException(
261+
String.format("failed to init database: %s", dataBaseName), ioe);
321262
}
263+
264+
logger.debug("Init DB {} done.", dataBaseName);
322265
} finally {
323266
resetDbLock.writeLock().unlock();
324267
}
@@ -523,7 +466,8 @@ public boolean deleteDbBakPath(String dir) {
523466

524467
@Override
525468
public RocksDbDataSourceImpl newInstance() {
526-
return new RocksDbDataSourceImpl(parentPath, dataBaseName, RocksDbSettings.getSettings());
469+
return new RocksDbDataSourceImpl(parentPath, dataBaseName,
470+
this.options);
527471
}
528472

529473

‎chainbase/src/main/java/org/tron/common/utils/StorageUtils.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import org.apache.commons.lang3.StringUtils;
77
import org.iq80.leveldb.Options;
88
import org.tron.common.parameter.CommonParameter;
9+
import org.tron.core.Constant;
910

1011

1112
public class StorageUtils {
@@ -52,9 +53,15 @@ public static String getOutputDirectory() {
5253
}
5354

5455
public static Options getOptionsByDbName(String dbName) {
56+
Options options;
5557
if (hasProperty(dbName)) {
56-
return getProperty(dbName).getDbOptions();
58+
options = getProperty(dbName).getDbOptions();
59+
} else {
60+
options = CommonParameter.getInstance().getStorage().newDefaultDbOptions(dbName);
5761
}
58-
return CommonParameter.getInstance().getStorage().newDefaultDbOptions(dbName);
62+
if (Constant.MARKET_PAIR_PRICE_TO_ORDER.equals(dbName)) {
63+
options.comparator(new MarketOrderPriceComparatorForLevelDB());
64+
}
65+
return options;
5966
}
6067
}

‎chainbase/src/main/java/org/tron/core/capsule/utils/MarketUtils.java‎

Lines changed: 4 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.tron.common.crypto.Hash;
2626
import org.tron.common.utils.ByteArray;
2727
import org.tron.common.utils.ByteUtil;
28+
import org.tron.common.utils.MarketComparator;
2829
import org.tron.core.capsule.AccountCapsule;
2930
import org.tron.core.capsule.MarketAccountOrderCapsule;
3031
import org.tron.core.capsule.MarketOrderCapsule;
@@ -227,29 +228,6 @@ public static byte[] createPairKey(byte[] sellTokenId, byte[] buyTokenId) {
227228
return result;
228229
}
229230

230-
/**
231-
* Note: the params should be the same token pair, or you should change the order.
232-
* All the quantity should be bigger than 0.
233-
* */
234-
public static int comparePrice(long price1SellQuantity, long price1BuyQuantity,
235-
long price2SellQuantity, long price2BuyQuantity) {
236-
try {
237-
return Long.compare(multiplyExact(price1BuyQuantity, price2SellQuantity, true),
238-
multiplyExact(price2BuyQuantity, price1SellQuantity, true));
239-
240-
} catch (ArithmeticException ex) {
241-
// do nothing here, because we will use BigInteger to compute again
242-
}
243-
244-
BigInteger price1BuyQuantityBI = BigInteger.valueOf(price1BuyQuantity);
245-
BigInteger price1SellQuantityBI = BigInteger.valueOf(price1SellQuantity);
246-
BigInteger price2BuyQuantityBI = BigInteger.valueOf(price2BuyQuantity);
247-
BigInteger price2SellQuantityBI = BigInteger.valueOf(price2SellQuantity);
248-
249-
return price1BuyQuantityBI.multiply(price2SellQuantityBI)
250-
.compareTo(price2BuyQuantityBI.multiply(price1SellQuantityBI));
251-
}
252-
253231
/**
254232
* if takerPrice >= makerPrice, return True
255233
* note: here are two different token pairs
@@ -265,7 +243,8 @@ public static boolean priceMatch(MarketPrice takerPrice, MarketPrice makerPrice)
265243
// ==> Price_TRX * sellQuantity_taker/buyQuantity_taker >= Price_TRX * buyQuantity_maker/sellQuantity_maker
266244
// ==> sellQuantity_taker * sellQuantity_maker > buyQuantity_taker * buyQuantity_maker
267245

268-
return comparePrice(takerPrice.getBuyTokenQuantity(), takerPrice.getSellTokenQuantity(),
246+
return MarketComparator.comparePrice(takerPrice.getBuyTokenQuantity(),
247+
takerPrice.getSellTokenQuantity(),
269248
makerPrice.getSellTokenQuantity(), makerPrice.getBuyTokenQuantity()) >= 0;
270249
}
271250

@@ -316,57 +295,7 @@ public static void returnSellTokenRemain(MarketOrderCapsule orderCapsule,
316295
}
317296

318297
public static int comparePriceKey(byte[] o1, byte[] o2) {
319-
//compare pair
320-
byte[] pair1 = new byte[TOKEN_ID_LENGTH * 2];
321-
byte[] pair2 = new byte[TOKEN_ID_LENGTH * 2];
322-
323-
System.arraycopy(o1, 0, pair1, 0, TOKEN_ID_LENGTH * 2);
324-
System.arraycopy(o2, 0, pair2, 0, TOKEN_ID_LENGTH * 2);
325-
326-
int pairResult = org.bouncycastle.util.Arrays.compareUnsigned(pair1, pair2);
327-
if (pairResult != 0) {
328-
return pairResult;
329-
}
330-
331-
//compare price
332-
byte[] getSellTokenQuantity1 = new byte[8];
333-
byte[] getBuyTokenQuantity1 = new byte[8];
334-
335-
byte[] getSellTokenQuantity2 = new byte[8];
336-
byte[] getBuyTokenQuantity2 = new byte[8];
337-
338-
int longByteNum = 8;
339-
340-
System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH,
341-
getSellTokenQuantity1, 0, longByteNum);
342-
System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum,
343-
getBuyTokenQuantity1, 0, longByteNum);
344-
345-
System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH,
346-
getSellTokenQuantity2, 0, longByteNum);
347-
System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum,
348-
getBuyTokenQuantity2, 0, longByteNum);
349-
350-
long sellTokenQuantity1 = ByteArray.toLong(getSellTokenQuantity1);
351-
long buyTokenQuantity1 = ByteArray.toLong(getBuyTokenQuantity1);
352-
long sellTokenQuantity2 = ByteArray.toLong(getSellTokenQuantity2);
353-
long buyTokenQuantity2 = ByteArray.toLong(getBuyTokenQuantity2);
354-
355-
if ((sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0)
356-
&& (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0)) {
357-
return 0;
358-
}
359-
360-
if (sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) {
361-
return -1;
362-
}
363-
364-
if (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0) {
365-
return 1;
366-
}
367-
368-
return comparePrice(sellTokenQuantity1, buyTokenQuantity1,
369-
sellTokenQuantity2, buyTokenQuantity2);
298+
return MarketComparator.comparePriceKey(o1, o2);
370299

371300
}
372301

‎chainbase/src/main/java/org/tron/core/db/TronDatabase.java‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@
99
import lombok.Getter;
1010
import lombok.extern.slf4j.Slf4j;
1111
import org.iq80.leveldb.WriteOptions;
12-
import org.rocksdb.DirectComparator;
1312
import org.springframework.beans.factory.annotation.Autowired;
1413
import org.tron.common.parameter.CommonParameter;
14+
import org.tron.common.storage.OptionsPicker;
1515
import org.tron.common.storage.WriteOptionsWrapper;
1616
import org.tron.common.storage.leveldb.LevelDbDataSourceImpl;
1717
import org.tron.common.storage.metric.DbStatService;
@@ -24,7 +24,7 @@
2424
import org.tron.core.exception.ItemNotFoundException;
2525

2626
@Slf4j(topic = "DB")
27-
public abstract class TronDatabase<T> implements ITronChainBase<T> {
27+
public abstract class TronDatabase<T> extends OptionsPicker implements ITronChainBase<T> {
2828

2929
protected DbSourceInter<byte[]> dbSource;
3030
@Getter
@@ -51,8 +51,7 @@ protected TronDatabase(String dbName) {
5151
String parentName = Paths.get(StorageUtils.getOutputDirectoryByDbName(dbName),
5252
CommonParameter.getInstance().getStorage().getDbDirectory()).toString();
5353
dbSource =
54-
new RocksDbDataSourceImpl(parentName, dbName, CommonParameter.getInstance()
55-
.getRocksDBCustomSettings(), getDirectComparator());
54+
new RocksDbDataSourceImpl(parentName, dbName, getOptionsByDbNameForRocksDB(dbName));
5655
}
5756

5857
dbSource.initDB();
@@ -66,14 +65,6 @@ protected void init() {
6665
protected TronDatabase() {
6766
}
6867

69-
protected org.iq80.leveldb.Options getOptionsByDbNameForLevelDB(String dbName) {
70-
return StorageUtils.getOptionsByDbName(dbName);
71-
}
72-
73-
protected DirectComparator getDirectComparator() {
74-
return null;
75-
}
76-
7768
public DbSourceInter<byte[]> getDbSource() {
7869
return dbSource;
7970
}

0 commit comments

Comments
 (0)