From 78f55a03e2ead2c888259e9b22436e5ffd10f159 Mon Sep 17 00:00:00 2001 From: federico Date: Mon, 26 Jan 2026 14:19:49 +0800 Subject: [PATCH 01/11] update toolkit based java-tron-v4.8.1 --- tools/gradlew/build.gradle | 42 + tools/stress_test/build.gradle | 4 +- .../src/main/java/org/tron/BroadcastTx.java | 12 +- .../main/java/org/tron/CollectAddress.java | 5 +- tools/toolkit/build.gradle | 40 +- .../arm/org/tron/plugins/ArchiveManifest.java | 29 + .../java/arm/org/tron/plugins/DbArchive.java | 50 + .../{ => common}/org/tron/plugins/Db.java | 4 +- .../org/tron/plugins/DbConvert.java | 59 +- .../{ => common}/org/tron/plugins/DbCopy.java | 2 +- .../{ => common}/org/tron/plugins/DbLite.java | 14 +- .../{ => common}/org/tron/plugins/DbMove.java | 0 .../{ => common}/org/tron/plugins/DbRoot.java | 24 +- .../org/tron/plugins/Toolkit.java | 0 .../org/tron/plugins/utils/ByteArray.java | 2 +- .../org/tron/plugins/utils/CryptoUitls.java | 0 .../org/tron/plugins/utils/DBUtils.java | 60 +- .../org/tron/plugins/utils/FileUtils.java | 0 .../org/tron/plugins/utils/MarketUtils.java | 68 + .../org/tron/plugins/utils/MerkleRoot.java | 0 .../org/tron/plugins/utils/Sha256Hash.java | 0 .../tron/plugins/utils/db/DBInterface.java | 39 + .../org/tron/plugins/utils/db/DBIterator.java | 0 .../org/tron/plugins/utils/db/DbTool.java | 24 +- .../tron/plugins/utils/db/LevelDBImpl.java | 7 +- .../plugins/utils/db/LevelDBIterator.java | 0 .../tron/plugins/utils/db/RockDBIterator.java | 12 +- .../tron/plugins/utils/db/RocksDBImpl.java | 97 + .../main/java/org/tron/plugins/DbFork.java | 419 ----- .../main/java/org/tron/plugins/DbQuery.java | 702 ------- .../MarketOrderPriceComparatorForLevelDB.java | 28 - .../MarketOrderPriceComparatorForRockDB.java | 38 - .../java/org/tron/plugins/utils/Constant.java | 63 - .../utils/HttpSelfFormatFieldName.java | 283 --- .../org/tron/plugins/utils/JsonFormat.java | 1663 ----------------- .../org/tron/plugins/utils/MarketUtils.java | 148 -- .../tron/plugins/utils/db/DBInterface.java | 23 - .../tron/plugins/utils/db/RocksDBImpl.java | 69 - .../org/tron/plugins/ArchiveManifest.java | 5 +- .../{ => x86}/org/tron/plugins/DbArchive.java | 5 +- tools/toolkit/src/main/resources/fork.conf | 58 - tools/toolkit/src/main/resources/logback.xml | 2 +- tools/toolkit/src/main/resources/query.conf | 20 - .../java/org/tron/plugins/DbCopyTest.java | 19 +- .../java/org/tron/plugins/DbForkTest.java | 206 -- .../java/org/tron/plugins/DbMoveTest.java | 79 +- .../java/org/tron/plugins/DbRootTest.java | 21 +- .../test/java/org/tron/plugins/DbTest.java | 44 +- .../{ => leveldb}/ArchiveManifestTest.java | 46 +- .../plugins/{ => leveldb}/DbArchiveTest.java | 45 +- .../plugins/{ => leveldb}/DbConvertTest.java | 16 +- .../{ => leveldb}/DbLiteLevelDbTest.java | 3 +- .../{ => leveldb}/DbLiteLevelDbV2Test.java | 3 +- .../{ => rocksdb}/DbLiteRocksDbTest.java | 3 +- .../plugins/rocksdb/DbLiteRocksDbV2Test.java | 13 + .../src/test/resources/config-duplicate.conf | 2 + .../src/test/resources/config-localtest.conf | 20 +- tools/toolkit/src/test/resources/config.conf | 8 +- tools/toolkit/src/test/resources/fork.conf | 58 - 59 files changed, 670 insertions(+), 4036 deletions(-) create mode 100644 tools/toolkit/src/main/java/arm/org/tron/plugins/ArchiveManifest.java create mode 100644 tools/toolkit/src/main/java/arm/org/tron/plugins/DbArchive.java rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/Db.java (88%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/DbConvert.java (89%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/DbCopy.java (98%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/DbLite.java (99%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/DbMove.java (100%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/DbRoot.java (84%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/Toolkit.java (100%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/utils/ByteArray.java (97%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/utils/CryptoUitls.java (100%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/utils/DBUtils.java (70%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/utils/FileUtils.java (100%) create mode 100644 tools/toolkit/src/main/java/common/org/tron/plugins/utils/MarketUtils.java rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/utils/MerkleRoot.java (100%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/utils/Sha256Hash.java (100%) create mode 100644 tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBInterface.java rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/utils/db/DBIterator.java (100%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/utils/db/DbTool.java (85%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/utils/db/LevelDBImpl.java (83%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/utils/db/LevelDBIterator.java (100%) rename tools/toolkit/src/main/java/{ => common}/org/tron/plugins/utils/db/RockDBIterator.java (76%) create mode 100644 tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RocksDBImpl.java delete mode 100644 tools/toolkit/src/main/java/org/tron/plugins/DbFork.java delete mode 100644 tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java delete mode 100644 tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java delete mode 100644 tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java delete mode 100644 tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java delete mode 100644 tools/toolkit/src/main/java/org/tron/plugins/utils/HttpSelfFormatFieldName.java delete mode 100644 tools/toolkit/src/main/java/org/tron/plugins/utils/JsonFormat.java delete mode 100644 tools/toolkit/src/main/java/org/tron/plugins/utils/MarketUtils.java delete mode 100644 tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBInterface.java delete mode 100644 tools/toolkit/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java rename tools/toolkit/src/main/java/{ => x86}/org/tron/plugins/ArchiveManifest.java (98%) rename tools/toolkit/src/main/java/{ => x86}/org/tron/plugins/DbArchive.java (98%) delete mode 100644 tools/toolkit/src/main/resources/fork.conf delete mode 100644 tools/toolkit/src/main/resources/query.conf delete mode 100644 tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java rename tools/toolkit/src/test/java/org/tron/plugins/{ => leveldb}/ArchiveManifestTest.java (69%) rename tools/toolkit/src/test/java/org/tron/plugins/{ => leveldb}/DbArchiveTest.java (71%) rename tools/toolkit/src/test/java/org/tron/plugins/{ => leveldb}/DbConvertTest.java (76%) rename tools/toolkit/src/test/java/org/tron/plugins/{ => leveldb}/DbLiteLevelDbTest.java (76%) rename tools/toolkit/src/test/java/org/tron/plugins/{ => leveldb}/DbLiteLevelDbV2Test.java (76%) rename tools/toolkit/src/test/java/org/tron/plugins/{ => rocksdb}/DbLiteRocksDbTest.java (76%) create mode 100644 tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java delete mode 100644 tools/toolkit/src/test/resources/fork.conf diff --git a/tools/gradlew/build.gradle b/tools/gradlew/build.gradle index 6029ef7e..60970845 100644 --- a/tools/gradlew/build.gradle +++ b/tools/gradlew/build.gradle @@ -1,7 +1,49 @@ +import org.gradle.nativeplatform.platform.internal.Architectures +import org.gradle.internal.os.OperatingSystem + plugins { id 'java' } +def arch = System.getProperty("os.arch").toLowerCase() +def javaVersion = JavaVersion.current() +def isArm64 = Architectures.AARCH64.isAlias(arch) +def archSource = isArm64 ? "arm" : "x86" +def isMac = OperatingSystem.current().isMacOsX() + +ext.archInfo = [ + name : arch, + java : javaVersion, + isArm64 : isArm64, + sourceSets: [ + main: [ + java: [ + srcDirs: ["src/main/java/common", "src/main/java/${archSource}"] + ] + ], + test: [ + java: [ + srcDirs: ["src/test/java"] + ] + ] + ], + requires: [ + JavaVersion: isArm64 ? JavaVersion.VERSION_17 : JavaVersion.VERSION_1_8, + RocksdbVersion: isArm64 ? '9.7.4' : '5.15.10', + // https://github.com/grpc/grpc-java/issues/7690 + // https://github.com/grpc/grpc-java/pull/12319, Add support for macOS aarch64 with universal binary + // https://github.com/grpc/grpc-java/pull/11371 , 1.64.x is not supported CentOS 7. + ProtocGenVersion: isArm64 && isMac ? '1.76.0' : '1.60.0' + ], + VMOptions: isArm64 ? "${rootDir}/gradle/jdk17/java-tron.vmoptions" : "${rootDir}/gradle/java-tron.vmoptions" +] + +if (!archInfo.java.is(archInfo.requires.JavaVersion)) { + throw new GradleException("Java ${archInfo.requires.JavaVersion} is required for ${archInfo.name}. Detected version ${archInfo.java}") +} + +println "Building for architecture: ${archInfo.name}, Java version: ${archInfo.java}" + tasks.named('jar') { enabled = false } diff --git a/tools/stress_test/build.gradle b/tools/stress_test/build.gradle index 894c81a8..8c2108aa 100644 --- a/tools/stress_test/build.gradle +++ b/tools/stress_test/build.gradle @@ -7,7 +7,7 @@ plugins { } sourceCompatibility = JavaVersion.VERSION_1_8 -targetCompatibility = JavaVersion.VERSION_1_8 +targetCompatibility = JavaVersion.current() [compileJava, compileTestJava]*.options*.encoding = 'UTF-8' repositories { @@ -52,7 +52,7 @@ dependencies { testCompileOnly 'org.projectlombok:lombok:1.18.12' testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' - implementation('com.github.tronprotocol.java-tron:framework:GreatVoyage-v4.7.7') { + implementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.20') { exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' exclude group: "pull-parser", module: "pull-parser" exclude group: "ch.qos.logback", module: "logback-classic" diff --git a/tools/stress_test/src/main/java/org/tron/BroadcastTx.java b/tools/stress_test/src/main/java/org/tron/BroadcastTx.java index dbc0f95e..b543acc5 100644 --- a/tools/stress_test/src/main/java/org/tron/BroadcastTx.java +++ b/tools/stress_test/src/main/java/org/tron/BroadcastTx.java @@ -7,6 +7,7 @@ import java.nio.file.Paths; import java.util.concurrent.Callable; import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.units.qual.A; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; @@ -105,10 +106,19 @@ public Integer call() throws IOException, InterruptedException, IllegalException Args.getInstance().setNodeListenPort(nodeListenPort); logger.info("rpc.port: {}, node.listen.port {}", rpcPort, nodeListenPort); Args.getInstance().setOpenHistoryQueryWhenLiteFN(true); + Args.getInstance().setRpcEnable(true); + Args.getInstance().setRpcSolidityEnable(false); + Args.getInstance().setRpcPBFTEnable(false); + Args.getInstance().setFullNodeHttpEnable(false); + Args.getInstance().setSolidityNodeHttpEnable(false); + Args.getInstance().setPBFTHttpEnable(false); + Args.getInstance().setJsonRpcHttpFullNodeEnable(false); + Args.getInstance().setJsonRpcHttpSolidityNodeEnable(false); + Args.getInstance().setJsonRpcHttpPBFTNodeEnable(false); context = new TronApplicationContext(DefaultConfig.class); app = ApplicationFactory.create(context); - app.addService(context.getBean(RpcApiService.class)); +// app.addService(context.getBean(RpcApiService.class)); app.startup(); String url = String.format("%s:%d", "127.0.0.1", diff --git a/tools/stress_test/src/main/java/org/tron/CollectAddress.java b/tools/stress_test/src/main/java/org/tron/CollectAddress.java index 8f57d2c9..cbf47f75 100644 --- a/tools/stress_test/src/main/java/org/tron/CollectAddress.java +++ b/tools/stress_test/src/main/java/org/tron/CollectAddress.java @@ -1,6 +1,6 @@ package org.tron; -import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; +//import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; @@ -130,7 +130,8 @@ private Set getAddressListFromDB(String dbPath, int totalNumber) throws IOException, RocksDBException { Set addressList = new HashSet<>(); String srcDir = dbPath + File.separator + "database"; - DBInterface accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); +// DBInterface accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); + DBInterface accountStore = DbTool.getDB(srcDir, "account"); DBIterator iterator = accountStore.iterator(); for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { addressList.add(ByteString.copyFrom(iterator.getKey())); diff --git a/tools/toolkit/build.gradle b/tools/toolkit/build.gradle index 450a14ca..b19e1048 100644 --- a/tools/toolkit/build.gradle +++ b/tools/toolkit/build.gradle @@ -4,10 +4,13 @@ plugins { id 'application' id 'checkstyle' id 'com.github.johnrengelman.shadow' version '7.1.2' + id "org.sonarqube" version "2.6" } +def springVersion = "5.3.39" + sourceCompatibility = JavaVersion.VERSION_1_8 -targetCompatibility = JavaVersion.VERSION_1_8 +targetCompatibility = JavaVersion.current() [compileJava, compileTestJava]*.options*.encoding = 'UTF-8' repositories { @@ -29,23 +32,34 @@ shadowJar { mergeServiceFiles() } +sourceSets { + main { + java.srcDirs = rootProject.archInfo.sourceSets.main.java.srcDirs + } + test { + java.srcDirs = rootProject.archInfo.sourceSets.test.java.srcDirs + } +} + + dependencies { testImplementation 'junit:junit:4.13.2' - testImplementation('com.github.tronprotocol.java-tron:framework:GreatVoyage-v4.7.7') { + testImplementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.20') { exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' exclude group: "pull-parser", module: "pull-parser" } testImplementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') + implementation group: 'org.springframework', name: 'spring-context', version: "${springVersion}" implementation group: 'info.picocli', name: 'picocli', version: '4.6.3' implementation group: 'com.typesafe', name: 'config', version: '1.3.2' implementation group: 'me.tongfei', name: 'progressbar', version: '0.9.3' - implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' + implementation group: 'org.bouncycastle', name: 'bcprov-jdk18on', version: '1.79' implementation group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10' implementation 'io.github.tronprotocol:leveldbjni-all:1.18.2' implementation 'io.github.tronprotocol:leveldb:1.18.2' - implementation('com.github.tronprotocol.java-tron:chainbase:GreatVoyage-v4.7.7') { + implementation('com.github.federico2014.java-tron:chainbase:GreatVoyage-v4.8.20') { exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' exclude group: "pull-parser", module: "pull-parser" } @@ -58,6 +72,24 @@ dependencies { testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' implementation 'com.alibaba:fastjson:1.2.83' + implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25' + implementation group: 'org.slf4j', name: 'jcl-over-slf4j', version: '1.7.25' + implementation "org.apache.commons:commons-lang3:3.4" + implementation "org.apache.commons:commons-collections4:4.1" + + if (rootProject.archInfo.isArm64) { + testRuntimeOnly group: 'org.fusesource.hawtjni', name: 'hawtjni-runtime', version: '1.18' // for test + implementation('com.github.federico2014.java-tron:platform:GreatVoyage-v4.8.20') + } else { + implementation('com.github.federico2014.java-tron:platform:GreatVoyage-v4.8.20') { + exclude(group: 'org.fusesource.leveldbjni', module: 'leveldbjni-all') + exclude(group: 'io.github.tronprotocol', module: 'zksnark-java-sdk') + exclude(group: 'commons-io', module: 'commons-io') + } + implementation 'io.github.tronprotocol:leveldbjni-all:1.18.2' + implementation 'io.github.tronprotocol:leveldb:1.18.2' + } + } application { diff --git a/tools/toolkit/src/main/java/arm/org/tron/plugins/ArchiveManifest.java b/tools/toolkit/src/main/java/arm/org/tron/plugins/ArchiveManifest.java new file mode 100644 index 00000000..b7848cf4 --- /dev/null +++ b/tools/toolkit/src/main/java/arm/org/tron/plugins/ArchiveManifest.java @@ -0,0 +1,29 @@ +package org.tron.plugins; + +import lombok.extern.slf4j.Slf4j; +import org.tron.common.arch.Arch; + +/** + * ARM architecture only supports RocksDB, + * which does not require manifest rebuilding (manifest rebuilding is a LevelDB-only feature). + * This command is not supported but retained for compatibility. + **/ +@Slf4j(topic = "archive") +public class ArchiveManifest { + + public static void main(String[] args) { + int exitCode = run(args); + System.exit(exitCode); + } + + public static int run(String[] args) { + String tips = String.format( + "%s architecture only supports RocksDB, which does not require manifest rebuilding " + + "(manifest rebuilding is a LevelDB-only feature).", + Arch.getOsArch()); + System.out.println(tips); + logger.warn(tips); + return 0; + } + +} diff --git a/tools/toolkit/src/main/java/arm/org/tron/plugins/DbArchive.java b/tools/toolkit/src/main/java/arm/org/tron/plugins/DbArchive.java new file mode 100644 index 00000000..03c52f33 --- /dev/null +++ b/tools/toolkit/src/main/java/arm/org/tron/plugins/DbArchive.java @@ -0,0 +1,50 @@ +package org.tron.plugins; + +import java.util.concurrent.Callable; +import lombok.extern.slf4j.Slf4j; +import org.tron.common.arch.Arch; +import picocli.CommandLine; +import picocli.CommandLine.Option; + +/** + * ARM architecture only supports RocksDB, + * which does not require manifest rebuilding (manifest rebuilding is a LevelDB-only feature). + * This command is not supported but retained for compatibility. + **/ +@Slf4j(topic = "archive") +@CommandLine.Command(name = "archive", description = "A helper to rewrite leveldb manifest.") +public class DbArchive implements Callable { + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + @Option(names = {"-d", "--database-directory"}, + defaultValue = "output-directory/database", + description = "java-tron database directory. Default: ${DEFAULT-VALUE}") + private String databaseDirectory; + + @Option(names = {"-b", "--batch-size"}, + defaultValue = "80000", + description = "deal manifest batch size. Default: ${DEFAULT-VALUE}") + private int maxBatchSize; + + @Option(names = {"-m", "--manifest-size"}, + defaultValue = "0", + description = "manifest min size(M) to archive. Default: ${DEFAULT-VALUE}") + private int maxManifestSize; + + @Option(names = {"-h", "--help"}) + private boolean help; + + + @Override + public Integer call() throws Exception { + String tips = String.format( + "%s architecture only supports RocksDB, which does not require manifest rebuilding " + + "(manifest rebuilding is a LevelDB-only feature).", + Arch.getOsArch()); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme().errorText(tips)); + logger.warn(tips); + return 0; + } + +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/Db.java b/tools/toolkit/src/main/java/common/org/tron/plugins/Db.java similarity index 88% rename from tools/toolkit/src/main/java/org/tron/plugins/Db.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/Db.java index 298718a9..84654dca 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/Db.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/Db.java @@ -12,9 +12,7 @@ DbConvert.class, DbLite.class, DbCopy.class, - DbRoot.class, - DbFork.class, - DbQuery.class + DbRoot.class }, commandListHeading = "%nCommands:%n%nThe most commonly used db commands are:%n" ) diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbConvert.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbConvert.java similarity index 89% rename from tools/toolkit/src/main/java/org/tron/plugins/DbConvert.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbConvert.java index a75b235b..bcf6e1e7 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbConvert.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbConvert.java @@ -20,6 +20,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.Status; +import org.tron.common.arch.Arch; import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.FileUtils; import picocli.CommandLine; @@ -49,20 +50,19 @@ public class DbConvert implements Callable { description = "Output path for rocksdb. Default: ${DEFAULT-VALUE}") private File dest; - @CommandLine.Option(names = {"--safe"}, - description = "In safe mode, read data from leveldb then put rocksdb." - + "If not, just change engine.properties from leveldb to rocksdb," - + "rocksdb is compatible with leveldb for current version." - + "This may not be the case in the future." - + "Default: ${DEFAULT-VALUE}") - private boolean safe; - @CommandLine.Option(names = {"-h", "--help"}) private boolean help; @Override public Integer call() throws Exception { + if (Arch.isArm64()) { + String tips = String.format("This command is not supported on %s architecture.", + Arch.getOsArch()); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme().errorText(tips)); + logger.error(tips); + return 0; + } if (help) { spec.commandLine().usage(System.out); return 0; @@ -95,12 +95,12 @@ public Integer call() throws Exception { final long time = System.currentTimeMillis(); List services = new ArrayList<>(); files.forEach(f -> services.add( - new DbConverter(src.getPath(), dest.getPath(), f.getName(), safe))); + new DbConverter(src.getPath(), dest.getPath(), f.getName()))); cpList.forEach(f -> services.add( new DbConverter( Paths.get(src.getPath(), DBUtils.CHECKPOINT_DB_V2).toString(), Paths.get(dest.getPath(), DBUtils.CHECKPOINT_DB_V2).toString(), - f.getName(), safe))); + f.getName()))); List fails = ProgressBar.wrap(services.stream(), "convert task").parallel().map( dbConverter -> { try { @@ -140,15 +140,12 @@ static class DbConverter implements Converter { private long srcDbValueSum = 0L; private long dstDbValueSum = 0L; - private boolean safe; - - public DbConverter(String srcDir, String dstDir, String name, boolean safe) { + public DbConverter(String srcDir, String dstDir, String name) { this.srcDir = srcDir; this.dstDir = dstDir; this.dbName = name; this.srcDbPath = Paths.get(this.srcDir, name); this.dstDbPath = Paths.get(this.dstDir, name); - this.safe = safe; } @Override @@ -178,23 +175,15 @@ public boolean doConvert() throws Exception { FileUtils.createDirIfNotExists(dstDir); logger.info("Convert database {} start", this.dbName); - if (safe) { - convertLevelToRocks(); - compact(); - } else { - FileUtils.copyDir(Paths.get(srcDir), Paths.get(dstDir), dbName); - } + convertLevelToRocks(); + compact(); + boolean result = check() && createEngine(dstDbPath.toString()); long etime = System.currentTimeMillis(); if (result) { - if (safe) { - logger.info("Convert database {} successful end with {} key-value {} minutes", - this.dbName, this.srcDbKeyCount, (etime - startTime) / 1000.0 / 60); - } else { - logger.info("Convert database {} successful end {} minutes", - this.dbName, (etime - startTime) / 1000.0 / 60); - } + logger.info("Convert database {} successful end with {} key-value {} minutes", + this.dbName, this.srcDbKeyCount, (etime - startTime) / 1000.0 / 60); } else { logger.info("Convert database {} failure", this.dbName); @@ -234,8 +223,8 @@ private void batchInsert(RocksDB rocks, List keys, List values) * @throws Exception RocksDBException */ private void write(RocksDB rocks, org.rocksdb.WriteBatch batch) throws Exception { - try { - rocks.write(new org.rocksdb.WriteOptions(), batch); + try (org.rocksdb.WriteOptions writeOptions = new org.rocksdb.WriteOptions()) { + rocks.write(writeOptions, batch); } catch (RocksDBException e) { // retry if (maybeRetry(e)) { @@ -270,7 +259,8 @@ public void convertLevelToRocks() throws Exception { JniDBFactory.pushMemoryPool(1024 * 1024); try ( DB level = DBUtils.newLevelDb(srcDbPath); - RocksDB rocks = DBUtils.newRocksDbForBulkLoad(dstDbPath); + org.rocksdb.Options options = DBUtils.newDefaultRocksDbOptions(true, dbName); + RocksDB rocks = RocksDB.open(options, this.dstDbPath.toString()); DBIterator levelIterator = level.iterator( new org.iq80.leveldb.ReadOptions().fillCache(false))) { @@ -302,7 +292,8 @@ private void compact() throws RocksDBException { if (DBUtils.MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(this.dbName)) { return; } - try (RocksDB rocks = DBUtils.newRocksDb(this.dstDbPath)) { + try (org.rocksdb.Options options = DBUtils.newDefaultRocksDbOptions(false, dbName); + RocksDB rocks = RocksDB.open(options, this.dstDbPath.toString())) { logger.info("compact database {} start", this.dbName); rocks.compactRange(); logger.info("compact database {} end", this.dbName); @@ -310,11 +301,9 @@ private void compact() throws RocksDBException { } private boolean check() throws RocksDBException { - if (!safe) { - return true; - } try ( - RocksDB rocks = DBUtils.newRocksDbReadOnly(this.dstDbPath); + org.rocksdb.Options options = DBUtils.newDefaultRocksDbOptions(false, dbName); + RocksDB rocks = RocksDB.openReadOnly(options, this.dstDbPath.toString()); org.rocksdb.ReadOptions r = new org.rocksdb.ReadOptions().setFillCache(false); RocksIterator rocksIterator = rocks.newIterator(r)) { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbCopy.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbCopy.java similarity index 98% rename from tools/toolkit/src/main/java/org/tron/plugins/DbCopy.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbCopy.java index 5a4e5969..7558d764 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbCopy.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbCopy.java @@ -22,7 +22,7 @@ exitCodeListHeading = "Exit Codes:%n", exitCodeList = { "0:Successful", - "n:Internal error: exception occurred, please check toolkit.log"}) + "n:Internal error: exception occurred,please check toolkit.log"}) public class DbCopy implements Callable { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbLite.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java similarity index 99% rename from tools/toolkit/src/main/java/org/tron/plugins/DbLite.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java index eb11883c..3f8a6cb5 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbLite.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java @@ -656,12 +656,13 @@ private boolean isLite(String databaseDir) throws RocksDBException, IOException private long getSecondBlock(String databaseDir) throws RocksDBException, IOException { long num = 0; DBInterface sourceBlockIndexDb = DbTool.getDB(databaseDir, BLOCK_INDEX_DB_NAME); - DBIterator iterator = sourceBlockIndexDb.iterator(); - iterator.seek(ByteArray.fromLong(1)); - if (iterator.hasNext()) { - num = Longs.fromByteArray(iterator.getKey()); + try (DBIterator iterator = sourceBlockIndexDb.iterator()) { + iterator.seek(ByteArray.fromLong(1)); + if (iterator.hasNext()) { + num = Longs.fromByteArray(iterator.getKey()); + } + return num; } - return num; } private DBInterface getCheckpointDb(String sourceDir) throws IOException, RocksDBException { @@ -720,3 +721,6 @@ public long getSnapshotMaxNum() { } } } + + + diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbMove.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbMove.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/DbMove.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbMove.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbRoot.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbRoot.java similarity index 84% rename from tools/toolkit/src/main/java/org/tron/plugins/DbRoot.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbRoot.java index 7c33219e..45854bbe 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbRoot.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbRoot.java @@ -67,16 +67,24 @@ public Integer call() throws Exception { .errorText("Specify at least one exit database: --db dbName.")); return 404; } - List task = ProgressBar.wrap(dbs.stream(), "root task").parallel() - .map(this::calcMerkleRoot).collect(Collectors.toList()); - task.forEach(this::printInfo); - int code = (int) task.stream().filter(r -> r.code == 1).count(); - if (code > 0) { + try { + List task = ProgressBar.wrap(dbs.stream(), "root task").parallel() + .map(this::calcMerkleRoot).collect(Collectors.toList()); + task.forEach(this::printInfo); + int code = (int) task.stream().filter(r -> r.code == 1).count(); + if (code > 0) { + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .errorText("There are some errors, please check toolkit.log for detail.")); + } + spec.commandLine().getOut().println("root task done."); + return code; + } catch (Exception e) { + logger.error("{}", e); spec.commandLine().getErr().println(spec.commandLine().getColorScheme() - .errorText("There are some errors, please check toolkit.log for detail.")); + .errorText(e.getMessage())); + spec.commandLine().usage(System.out); + return 1; } - spec.commandLine().getOut().println("root task done."); - return code; } private Ret calcMerkleRoot(String name) { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/Toolkit.java b/tools/toolkit/src/main/java/common/org/tron/plugins/Toolkit.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/Toolkit.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/Toolkit.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/ByteArray.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/ByteArray.java similarity index 97% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/ByteArray.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/ByteArray.java index 3422c36c..2922d110 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/ByteArray.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/ByteArray.java @@ -59,7 +59,7 @@ public static int compareUnsigned(byte[] a, byte[] b) { if (b == null) { return 1; } - int minLen = Math.min(a.length, b.length); + int minLen = StrictMath.min(a.length, b.length); for (int i = 0; i < minLen; ++i) { int aVal = a[i] & 0xFF; int bVal = b[i] & 0xFF; diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/CryptoUitls.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/CryptoUitls.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/CryptoUitls.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/CryptoUitls.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/DBUtils.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/DBUtils.java similarity index 70% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/DBUtils.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/DBUtils.java index 19547e2b..6eb097cb 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/DBUtils.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/DBUtils.java @@ -14,10 +14,9 @@ import org.rocksdb.BloomFilter; import org.rocksdb.ComparatorOptions; import org.rocksdb.Options; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; -import org.tron.plugins.comparator.MarketOrderPriceComparatorForLevelDB; -import org.tron.plugins.comparator.MarketOrderPriceComparatorForRockDB; +import org.tron.common.arch.Arch; +import org.tron.common.utils.MarketOrderPriceComparatorForLevelDB; +import org.tron.common.utils.MarketOrderPriceComparatorForRocksDB; import org.tron.protos.Protocol; public class DBUtils { @@ -65,6 +64,7 @@ static Operator valueOf(byte b) { public static final String ROCKSDB = "ROCKSDB"; public static DB newLevelDb(Path db) throws IOException { + Arch.throwIfUnsupportedArm64Exception(LEVELDB); File file = db.toFile(); org.iq80.leveldb.Options dbOptions = newDefaultLevelDbOptions(); if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(file.getName())) { @@ -86,7 +86,23 @@ public static org.iq80.leveldb.Options newDefaultLevelDbOptions() { return dbOptions; } - private static Options newDefaultRocksDbOptions(boolean forBulkLoad) { + /** + * Creates a new RocksDB Options. + * + *

CRITICAL: Must be closed after use to prevent native memory leaks. + * Use try-with-resources. + * + * 

{@code
+   * try (Options options = newDefaultRocksDbOptions(false, name)) {
+   *     // do something
+   * }
+   * }
+ * + * @param forBulkLoad if true, optimizes for bulk loading + * @param name db name + * @return a new Options instance that must be closed + */ + public static Options newDefaultRocksDbOptions(boolean forBulkLoad, String name) { Options options = new Options(); options.setCreateIfMissing(true); options.setIncreaseParallelism(1); @@ -95,7 +111,8 @@ private static Options newDefaultRocksDbOptions(boolean forBulkLoad) { options.setTargetFileSizeBase(64 * 1024 * 1024); options.setTargetFileSizeMultiplier(1); options.setMaxBytesForLevelBase(512 * 1024 * 1024); - options.setMaxBackgroundCompactions(Math.max(1, Runtime.getRuntime().availableProcessors())); + options.setMaxBackgroundCompactions(StrictMath.max( + 1, Runtime.getRuntime().availableProcessors())); options.setLevel0FileNumCompactionTrigger(4); options.setLevelCompactionDynamicLevelBytes(true); final BlockBasedTableConfig tableCfg; @@ -108,35 +125,10 @@ private static Options newDefaultRocksDbOptions(boolean forBulkLoad) { if (forBulkLoad) { options.prepareForBulkLoad(); } - return options; - } - - public static RocksDB newRocksDb(Path db) throws RocksDBException { - try (Options options = newDefaultRocksDbOptions(false)) { - if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); - } - return RocksDB.open(options, db.toString()); - } - } - - public static RocksDB newRocksDbForBulkLoad(Path db) throws RocksDBException { - try (Options options = newDefaultRocksDbOptions(true)) { - if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); - } - return RocksDB.open(options, db.toString()); - } - } - - - public static RocksDB newRocksDbReadOnly(Path db) throws RocksDBException { - try (Options options = newDefaultRocksDbOptions(false)) { - if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); - } - return RocksDB.openReadOnly(options, db.toString()); + if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(name)) { + options.setComparator(new MarketOrderPriceComparatorForRocksDB(new ComparatorOptions())); } + return options; } public static String simpleDecode(byte[] bytes) { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/FileUtils.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/FileUtils.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/FileUtils.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/FileUtils.java diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/MarketUtils.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/MarketUtils.java new file mode 100644 index 00000000..9bcfd5e7 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/MarketUtils.java @@ -0,0 +1,68 @@ +package org.tron.plugins.utils; + +public class MarketUtils { + + public static final int TOKEN_ID_LENGTH = ByteArray + .fromString(Long.toString(Long.MAX_VALUE)).length; // 19 + + + + /** + * In order to avoid the difference between the data of same key stored and fetched by hashMap and + * levelDB, when creating the price key, we will find the GCD (Greatest Common Divisor) of + * sellTokenQuantity and buyTokenQuantity. + */ + public static byte[] createPairPriceKey(byte[] sellTokenId, byte[] buyTokenId, + long sellTokenQuantity, long buyTokenQuantity) { + + byte[] sellTokenQuantityBytes; + byte[] buyTokenQuantityBytes; + + // cal the GCD + long gcd = findGCD(sellTokenQuantity, buyTokenQuantity); + if (gcd == 0) { + sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity); + buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity); + } else { + sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity / gcd); + buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity / gcd); + } + + return doCreatePairPriceKey(sellTokenId, buyTokenId, + sellTokenQuantityBytes, buyTokenQuantityBytes); + } + + public static long findGCD(long number1, long number2) { + if (number1 == 0 || number2 == 0) { + return 0; + } + return calGCD(number1, number2); + } + + private static long calGCD(long number1, long number2) { + if (number2 == 0) { + return number1; + } + return calGCD(number2, number1 % number2); + } + + + private static byte[] doCreatePairPriceKey(byte[] sellTokenId, byte[] buyTokenId, + byte[] sellTokenQuantity, byte[] buyTokenQuantity) { + byte[] result = new byte[TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + + sellTokenQuantity.length + buyTokenQuantity.length]; + + System.arraycopy(sellTokenId, 0, result, 0, sellTokenId.length); + System.arraycopy(buyTokenId, 0, result, TOKEN_ID_LENGTH, buyTokenId.length); + System.arraycopy(sellTokenQuantity, 0, result, + TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, + sellTokenQuantity.length); + System.arraycopy(buyTokenQuantity, 0, result, + TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + buyTokenQuantity.length, + buyTokenQuantity.length); + + return result; + } + + +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/MerkleRoot.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/MerkleRoot.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/MerkleRoot.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/MerkleRoot.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/Sha256Hash.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/Sha256Hash.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/Sha256Hash.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/Sha256Hash.java diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBInterface.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBInterface.java new file mode 100644 index 00000000..13a195f9 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBInterface.java @@ -0,0 +1,39 @@ +package org.tron.plugins.utils.db; + +import java.io.Closeable; +import java.io.IOException; + + +public interface DBInterface extends Closeable { + + byte[] get(byte[] key); + + void put(byte[] key, byte[] value); + + void delete(byte[] key); + + /** + * Returns an iterator over the database. + * + *

CRITICAL: The returned iterator holds native resources and MUST be closed + * after use to prevent memory leaks. It is strongly recommended to use a try-with-resources + * statement. + * + *

Example of correct usage: + * 

{@code
+   * try (DBIterator iterator = db.iterator()) {
+   *  // do something
+   * }
+   * }
+ * + * @return a new database iterator that must be closed. + */ + DBIterator iterator(); + + long size() throws IOException; + + void close() throws IOException; + + String getName(); + +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBIterator.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBIterator.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBIterator.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBIterator.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DbTool.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java similarity index 85% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/DbTool.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java index 429025e8..127b8f97 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DbTool.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java @@ -18,8 +18,8 @@ public class DbTool { private static final String KEY_ENGINE = "ENGINE"; private static final String ENGINE_FILE = "engine.properties"; - private static final String FILE_SEPARATOR = File.separator; private static final String ROCKSDB = "ROCKSDB"; + private static final String LEVELDB = "LEVELDB"; private static final Map dbMap = Maps.newConcurrentMap(); @@ -162,8 +162,7 @@ public static void close() { } private static DbType getDbType(String sourceDir, String dbName) { - String engineFile = String.format("%s%s%s%s%s", sourceDir, FILE_SEPARATOR, - dbName, FILE_SEPARATOR, ENGINE_FILE); + String engineFile = Paths.get(sourceDir, dbName, ENGINE_FILE).toString(); if (!new File(engineFile).exists()) { return DbType.LevelDB; } @@ -175,13 +174,22 @@ private static DbType getDbType(String sourceDir, String dbName) { } } - private static LevelDBImpl openLevelDb(Path db, String name) throws IOException { - return new LevelDBImpl(DBUtils.newLevelDb(db), name); + public static LevelDBImpl openLevelDb(Path db, String name) throws IOException { + LevelDBImpl leveldb = new LevelDBImpl(DBUtils.newLevelDb(db), name); + tryInitEngineFile(db, LEVELDB); + return leveldb; } - private static RocksDBImpl openRocksDb(Path db, String name) throws RocksDBException { - return new RocksDBImpl(DBUtils.newRocksDb(db), name); + public static RocksDBImpl openRocksDb(Path db, String name) throws RocksDBException { + RocksDBImpl rocksdb = new RocksDBImpl(db, name); + tryInitEngineFile(db, ROCKSDB); + return rocksdb; } - + private static void tryInitEngineFile(Path db, String engine) { + String engineFile = Paths.get(db.toString(), ENGINE_FILE).toString(); + if (FileUtils.createFileIfNotExists(engineFile)) { + FileUtils.writeProperty(engineFile, KEY_ENGINE, engine); + } + } } diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBImpl.java similarity index 83% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBImpl.java index 511f4dfd..1c7f22ef 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBImpl.java @@ -40,8 +40,11 @@ public DBIterator iterator() { } @Override - public long size() { - return Streams.stream(leveldb.iterator()).count(); + public long size() throws IOException { + try (DBIterator iterator = this.iterator()) { + iterator.seekToFirst(); + return Streams.stream(iterator).count(); + } } @Override diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBIterator.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBIterator.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBIterator.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBIterator.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RockDBIterator.java similarity index 76% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RockDBIterator.java index d3e17d91..17ecca4a 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RockDBIterator.java @@ -2,14 +2,19 @@ import java.io.IOException; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import org.rocksdb.ReadOptions; import org.rocksdb.RocksIterator; public class RockDBIterator implements DBIterator { private final RocksIterator iterator; + private final ReadOptions readOptions; + private final AtomicBoolean closed = new AtomicBoolean(false); - public RockDBIterator(RocksIterator iterator) { + public RockDBIterator(RocksIterator iterator, ReadOptions readOptions) { this.iterator = iterator; + this.readOptions = readOptions; } @Override @@ -72,6 +77,9 @@ public byte[] setValue(byte[] value) { @Override public void close() throws IOException { - iterator.close(); + if (closed.compareAndSet(false, true)) { + readOptions.close(); + iterator.close(); + } } } diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RocksDBImpl.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RocksDBImpl.java new file mode 100644 index 00000000..236d0a84 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RocksDBImpl.java @@ -0,0 +1,97 @@ +package org.tron.plugins.utils.db; + +import com.google.common.collect.Streams; +import java.io.IOException; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.Getter; +import org.rocksdb.Options; +import org.rocksdb.ReadOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.DBUtils; + +public class RocksDBImpl implements DBInterface { + + private final RocksDB rocksDB; + + @Getter + private final String name; + private final AtomicBoolean closed = new AtomicBoolean(false); + private Options options = null; + + public RocksDBImpl(Path path, String name) throws RocksDBException { + try { + this.options = DBUtils.newDefaultRocksDbOptions(false, name); + this.name = name; + this.rocksDB = RocksDB.open(options, path.toString()); + } catch (RocksDBException e) { + if (this.options != null) { + this.options.close(); + } + throw e; + } + } + + @Override + public byte[] get(byte[] key) { + throwIfClosed(); + try { + return rocksDB.get(key); + } catch (RocksDBException e) { + throw new RuntimeException(name, e); + } + } + + @Override + public void put(byte[] key, byte[] value) { + throwIfClosed(); + try { + rocksDB.put(key, value); + } catch (RocksDBException e) { + throw new RuntimeException(name, e); + } + } + + @Override + public void delete(byte[] key) { + throwIfClosed(); + try { + rocksDB.delete(key); + } catch (RocksDBException e) { + throw new RuntimeException(name, e); + } + } + + @Override + public DBIterator iterator() { + throwIfClosed(); + ReadOptions readOptions = new ReadOptions().setFillCache(false); + return new RockDBIterator(rocksDB.newIterator(readOptions), readOptions); + } + + @Override + public long size() throws IOException { + throwIfClosed(); + try (DBIterator iterator = this.iterator()) { + iterator.seekToFirst(); + return Streams.stream(iterator).count(); + } + } + + @Override + public void close() throws IOException { + if (closed.compareAndSet(false, true)) { + if (this.options != null) { + this.options.close(); + } + rocksDB.close(); + } + } + + private void throwIfClosed() { + if (closed.get()) { + throw new IllegalStateException("db " + name + " has been closed"); + } + } +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbFork.java b/tools/toolkit/src/main/java/org/tron/plugins/DbFork.java deleted file mode 100644 index 0e6919f7..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbFork.java +++ /dev/null @@ -1,419 +0,0 @@ -package org.tron.plugins; - -import static java.lang.System.arraycopy; -import static org.tron.plugins.utils.Constant.ACCOUNTS_KEY; -import static org.tron.plugins.utils.Constant.ACCOUNT_ADDRESS; -import static org.tron.plugins.utils.Constant.ACCOUNT_ASSET; -import static org.tron.plugins.utils.Constant.ACCOUNT_BALANCE; -import static org.tron.plugins.utils.Constant.ACCOUNT_NAME; -import static org.tron.plugins.utils.Constant.ACCOUNT_OWNER; -import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; -import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_BALANCE; -import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_ID; -import static org.tron.plugins.utils.Constant.ACCOUNT_TYPE; -import static org.tron.plugins.utils.Constant.ACTIVE_WITNESSES; -import static org.tron.plugins.utils.Constant.ASSET_ISSUE_V2; -import static org.tron.plugins.utils.Constant.CONTRACT_STORE; -import static org.tron.plugins.utils.Constant.DYNAMIC_PROPERTY_STORE; -import static org.tron.plugins.utils.Constant.LATEST_BLOCK_HEADER_TIMESTAMP; -import static org.tron.plugins.utils.Constant.LATEST_BLOCK_TIMESTAMP; -import static org.tron.plugins.utils.Constant.MAINTENANCE_INTERVAL; -import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME; -import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME_INTERVAL; -import static org.tron.plugins.utils.Constant.MAX_ACTIVE_WITNESS_NUM; -import static org.tron.plugins.utils.Constant.NEXT_MAINTENANCE_TIME; -import static org.tron.plugins.utils.Constant.STORAGE_ROW_STORE; -import static org.tron.plugins.utils.Constant.TRC20_ACCOUNT; -import static org.tron.plugins.utils.Constant.TRC20_BALANCE; -import static org.tron.plugins.utils.Constant.TRC20_BALANCES_POSITION; -import static org.tron.plugins.utils.Constant.TRC20_CONTRACTS_KEY; -import static org.tron.plugins.utils.Constant.TRC20_CONTRACT_ADDRESS; -import static org.tron.plugins.utils.Constant.WITNESS_ADDRESS; -import static org.tron.plugins.utils.Constant.WITNESS_KEY; -import static org.tron.plugins.utils.Constant.WITNESS_SCHEDULE_STORE; -import static org.tron.plugins.utils.Constant.WITNESS_STORE; -import static org.tron.plugins.utils.Constant.WITNESS_URL; -import static org.tron.plugins.utils.Constant.WITNESS_VOTE; - -import com.google.common.primitives.Bytes; -import com.google.common.primitives.Longs; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import java.io.File; -import java.io.IOException; -import java.math.BigInteger; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ArrayUtils; -import org.rocksdb.RocksDBException; -import org.tron.common.crypto.Hash; -import org.tron.common.utils.ByteArray; -import org.tron.common.utils.ByteUtil; -import org.tron.common.utils.Commons; -import org.tron.core.capsule.AccountCapsule; -import org.tron.core.capsule.StorageRowCapsule; -import org.tron.core.capsule.WitnessCapsule; -import org.tron.plugins.utils.Constant; -import org.tron.plugins.utils.FileUtils; -import org.tron.plugins.utils.db.DBInterface; -import org.tron.plugins.utils.db.DBIterator; -import org.tron.plugins.utils.db.DbTool; -import org.tron.protos.Protocol.Account; -import org.tron.protos.Protocol.AccountType; -import org.tron.protos.Protocol.Permission; -import org.tron.protos.contract.SmartContractOuterClass.SmartContract; -import picocli.CommandLine; -import picocli.CommandLine.Command; - -@Slf4j(topic = "fork") -@Command(name = "fork", - description = "Modify the database of java-tron for shadow fork testing.", - exitCodeListHeading = "Exit Codes:%n", - exitCodeList = { - "0:Successful", - "n:Internal error: exception occurred, please check logs/toolkit.log"}) -public class DbFork implements Callable { - - private DBInterface witnessStore; - private DBInterface witnessScheduleStore; - private DBInterface accountStore; - private DBInterface dynamicPropertiesStore; - private DBInterface accountAssetStore; - private DBInterface assetIssueV2Store; - private DBInterface contractStore; - private DBInterface storageRowStore; - - @CommandLine.Spec - CommandLine.Model.CommandSpec spec; - - @CommandLine.Option(names = {"-d", "--database-directory"}, - defaultValue = "output-directory", - description = "java-tron database directory path. Default: ${DEFAULT-VALUE}") - private String database; - - @CommandLine.Option(names = {"-c", "--config"}, - defaultValue = "fork.conf", - description = "config the new witnesses, balances, etc for shadow fork." - + " Default: ${DEFAULT-VALUE}") - private String config; - - @CommandLine.Option(names = {"-r", "--retain-witnesses"}, - description = "retain the previous witnesses and active witnesses.") - private boolean retain; - - @CommandLine.Option(names = {"-h", "--help"}) - private boolean help; - - private void initStore() throws IOException, RocksDBException { - String srcDir = database + File.separator + "database"; - witnessStore = DbTool.getDB(srcDir, WITNESS_STORE); - witnessScheduleStore = DbTool.getDB(srcDir, WITNESS_SCHEDULE_STORE); - accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); - dynamicPropertiesStore = DbTool.getDB(srcDir, DYNAMIC_PROPERTY_STORE); - accountAssetStore = DbTool.getDB(srcDir, ACCOUNT_ASSET); - assetIssueV2Store = DbTool.getDB(srcDir, ASSET_ISSUE_V2); - contractStore = DbTool.getDB(srcDir, CONTRACT_STORE); - storageRowStore = DbTool.getDB(srcDir, STORAGE_ROW_STORE); - } - - @Override - public Integer call() throws IOException, RocksDBException { - if (help) { - spec.commandLine().usage(System.out); - return 0; - } - - File dbFile = Paths.get(database).toFile(); - if (!dbFile.exists() || !dbFile.isDirectory()) { - logger.error("Database [" + database + "] not exists!"); - spec.commandLine().getErr().format("Database %s not exists!", database).println(); - System.exit(-1); - } - File tmp = Paths.get(database, "database", "tmp").toFile(); - if (tmp.exists()) { - FileUtils.deleteDir(tmp); - } - - Config forkConfig = ConfigFactory.load(); - File file = Paths.get(config).toFile(); - if (file.exists() && file.isFile()) { - forkConfig = ConfigFactory.parseFile(file); - } else { - logger.error("Fork config file [" + config + "] not exists!"); - spec.commandLine().getErr().format("Fork config file: %s not exists!", config).println(); - System.exit(-1); - } - - initStore(); - - if (!retain) { - logger.info("Erase the previous witnesses and active witnesses."); - spec.commandLine().getOut().println("Erase the previous witnesses and active witnesses."); - witnessScheduleStore.delete(ACTIVE_WITNESSES); - DBIterator iterator = witnessStore.iterator(); - for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { - witnessStore.delete(iterator.getKey()); - } - } else { - logger.warn("Keep the previous witnesses and active witnesses."); - spec.commandLine().getOut().println("Keep the previous witnesses and active witnesses."); - } - - if (forkConfig.hasPath(WITNESS_KEY)) { - List witnesses = forkConfig.getConfigList(WITNESS_KEY); - if (witnesses.isEmpty()) { - spec.commandLine().getOut().println("no witness listed in the config."); - } - witnesses = witnesses.stream() - .filter(c -> c.hasPath(WITNESS_ADDRESS)) - .collect(Collectors.toList()); - - if (witnesses.isEmpty()) { - spec.commandLine().getOut().println("no witness listed in the config."); - } - - List witnessList = new ArrayList<>(); - witnesses.stream().forEach( - w -> { - ByteString address = ByteString.copyFrom( - Commons.decodeFromBase58Check(w.getString(WITNESS_ADDRESS))); - WitnessCapsule witness = new WitnessCapsule(address); - witness.setIsJobs(true); - if (w.hasPath(WITNESS_VOTE) && w.getLong(WITNESS_VOTE) > 0) { - witness.setVoteCount(w.getLong(WITNESS_VOTE)); - } - if (w.hasPath(WITNESS_URL)) { - witness.setUrl(w.getString(WITNESS_URL)); - } - witnessStore.put(address.toByteArray(), witness.getData()); - witnessList.add(witness.getAddress()); - }); - - witnessList.sort(Comparator.comparingLong((ByteString b) -> - new WitnessCapsule(witnessStore.get(b.toByteArray())).getVoteCount()) - .reversed() - .thenComparing(Comparator.comparingInt(ByteString::hashCode).reversed())); - List activeWitnesses = witnessList.subList(0, - witnesses.size() >= MAX_ACTIVE_WITNESS_NUM ? MAX_ACTIVE_WITNESS_NUM : witnessList.size()); - witnessScheduleStore.put(ACTIVE_WITNESSES, getActiveWitness(activeWitnesses)); - logger.info("{} witnesses and {} active witnesses have been modified.", - witnesses.size(), activeWitnesses.size()); - spec.commandLine().getOut().format("%d witnesses and %d active witnesses have been modified.", - witnesses.size(), activeWitnesses.size()).println(); - } - - if (forkConfig.hasPath(ACCOUNTS_KEY)) { - List accounts = forkConfig.getConfigList(ACCOUNTS_KEY); - if (accounts.isEmpty()) { - spec.commandLine().getOut().println("no account listed in the config."); - } - - accounts = accounts.stream() - .filter(c -> c.hasPath(ACCOUNT_ADDRESS)) - .collect(Collectors.toList()); - - if (accounts.isEmpty()) { - spec.commandLine().getOut().println("no account listed in the config."); - } - - accounts.stream().forEach( - a -> { - byte[] address = Commons.decodeFromBase58Check(a.getString(ACCOUNT_ADDRESS)); - byte[] value = accountStore.get(address); - Account account = null; - try { - account = ArrayUtils.isEmpty(value) ? null : Account.parseFrom(value); - } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); - System.exit(-1); - } - - if (Objects.isNull(account)) { - ByteString byteAddress = ByteString.copyFrom( - Commons.decodeFromBase58Check(a.getString(ACCOUNT_ADDRESS))); - account = Account.newBuilder().setAddress(byteAddress).build(); - } - AccountCapsule accountCapsule = new AccountCapsule(account); - - if (a.hasPath(ACCOUNT_BALANCE) && a.getLong(ACCOUNT_BALANCE) > 0) { - accountCapsule.setBalance(a.getLong(ACCOUNT_BALANCE)); - } - if (a.hasPath(ACCOUNT_NAME)) { - accountCapsule.setAccountName( - ByteArray.fromString(a.getString(ACCOUNT_NAME))); - } - if (a.hasPath(ACCOUNT_TYPE)) { - accountCapsule.updateAccountType( - AccountType.valueOf(a.getString(ACCOUNT_TYPE))); - } - - if (a.hasPath(ACCOUNT_OWNER)) { - byte[] owner = Commons.decodeFromBase58Check(a.getString(ACCOUNT_OWNER)); - Permission ownerPermission = AccountCapsule - .createDefaultOwnerPermission(ByteString.copyFrom(owner)); - accountCapsule.updatePermissions(ownerPermission, null, null); - } - - if (a.hasPath(ACCOUNT_TRC10_ID) && a.hasPath(ACCOUNT_TRC10_BALANCE) - && a.getLong(ACCOUNT_TRC10_BALANCE) > 0) { - String trc10Id = a.getString(ACCOUNT_TRC10_ID); - if (assetIssueV2Store.get(ByteArray.fromString(trc10Id)) != null) { - if (accountCapsule.getAssetOptimized()) { - byte[] k = Bytes.concat(address, ByteArray.fromString(trc10Id)); - accountAssetStore.put(k, Longs.toByteArray(a.getLong(ACCOUNT_TRC10_BALANCE))); - } else { - Map assetMapV2 = new HashMap<>(account.getAssetV2Map()); - assetMapV2.put(trc10Id, a.getLong(ACCOUNT_TRC10_BALANCE)); - accountCapsule.clearAssetV2(); - accountCapsule.addAssetMapV2(assetMapV2); - } - } else { - logger.info("TRC10: {} not exists in the database.", trc10Id); - spec.commandLine().getOut().format("TRC10: %s not exists in the database.", trc10Id) - .println(); - } - } - - accountStore.put(address, accountCapsule.getData()); - }); - logger.info("{} accounts have been modified.", accounts.size()); - spec.commandLine().getOut().format("%d accounts have been modified.", accounts.size()) - .println(); - } - - if (forkConfig.hasPath(TRC20_CONTRACTS_KEY)) { - List trc20Contracts = forkConfig.getConfigList(TRC20_CONTRACTS_KEY); - if (trc20Contracts.isEmpty()) { - spec.commandLine().getOut().println("no TRC20 contract listed in the config."); - } - - trc20Contracts = trc20Contracts.stream() - .filter(c -> c.hasPath(TRC20_CONTRACT_ADDRESS) && c.hasPath(TRC20_BALANCES_POSITION) && - c.hasPath(TRC20_ACCOUNT) && c.hasPath(TRC20_BALANCE)) - .collect(Collectors.toList()); - - if (trc20Contracts.isEmpty()) { - spec.commandLine().getOut().println("no TRC20 contract listed in the config."); - } - - AtomicInteger cnt = new AtomicInteger(); - - trc20Contracts.stream().forEach( - contract -> { - byte[] contractAddress = Commons - .decodeFromBase58Check(contract.getString(TRC20_CONTRACT_ADDRESS)); - if (contractStore.get(contractAddress) == null) { - spec.commandLine().getErr().format("TRC20 contract: %s not exists in the database.", - contract.getString(TRC20_CONTRACT_ADDRESS)).println(); - return; - } - - SmartContract smartContract; - try { - smartContract = SmartContract.parseFrom(contractStore.get(contractAddress)); - } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); - return; - } - - int balancesSlotPosition = 0; - if (contract.getInt(TRC20_BALANCES_POSITION) > 0) { - balancesSlotPosition = contract.getInt(TRC20_BALANCES_POSITION); - } - byte[] addressWithPrefix = Commons - .decodeFromBase58Check(contract.getString(TRC20_ACCOUNT)); - byte[] address = ByteArray.subArray(addressWithPrefix, 1, 21); - String paddedAddress = String - .format("%064x", new BigInteger(ByteArray.toHexString(address), 16)); - String paddedSlot = String.format("%064x", balancesSlotPosition); - byte[] contractKey = Hash.sha3(ByteArray.fromHexString( - paddedAddress + paddedSlot)); - - byte[] addressHash; - byte[] trxHash = smartContract.getTrxHash().toByteArray(); - if (ByteUtil.isNullOrZeroArray(trxHash)) { - addressHash = Hash.sha3(contractAddress); - } else { - addressHash = Hash.sha3(ByteUtil.merge(contractAddress, trxHash)); - } - - int contractVersion = smartContract.getVersion(); - if (contractVersion == 1) { - contractKey = Hash.sha3(contractKey); - } - byte[] rowKey = new byte[contractKey.length]; - arraycopy(addressHash, 0, rowKey, 0, 16); - arraycopy(contractKey, 16, rowKey, 16, 16); - - String paddedBalance = String - .format("%064x", new BigInteger(contract.getString(TRC20_BALANCE), 10)); - byte[] rowValue = ByteArray.fromHexString(paddedBalance); - StorageRowCapsule storageRowCapsule = new StorageRowCapsule(rowKey, rowValue); - - storageRowStore.put(rowKey, storageRowCapsule.getData()); - cnt.getAndIncrement(); - }); - logger.info("{} TRC20 contracts have been modified.", cnt.get()); - spec.commandLine().getOut() - .format("%d TRC20 contracts have been modified.", cnt.get()) - .println(); - } - - if (forkConfig.hasPath(LATEST_BLOCK_TIMESTAMP) - && forkConfig.getLong(LATEST_BLOCK_TIMESTAMP) > 0) { - long latestBlockHeaderTimestamp = forkConfig.getLong(LATEST_BLOCK_TIMESTAMP); - dynamicPropertiesStore - .put(LATEST_BLOCK_HEADER_TIMESTAMP, ByteArray.fromLong(latestBlockHeaderTimestamp)); - logger.info("The latest block header timestamp has been modified as {}.", - latestBlockHeaderTimestamp); - spec.commandLine().getOut().format("The latest block header timestamp has been modified " - + "as %d.", latestBlockHeaderTimestamp).println(); - } - - if (forkConfig.hasPath(MAINTENANCE_INTERVAL) - && forkConfig.getLong(MAINTENANCE_INTERVAL) > 0) { - long maintenanceTimeInterval = forkConfig.getLong(MAINTENANCE_INTERVAL); - dynamicPropertiesStore - .put(MAINTENANCE_TIME_INTERVAL, ByteArray.fromLong(maintenanceTimeInterval)); - logger.info("The maintenance time interval has been modified as {}.", - maintenanceTimeInterval); - spec.commandLine().getOut().format("The maintenance time interval has been modified as %d.", - maintenanceTimeInterval).println(); - } - - if (forkConfig.hasPath(NEXT_MAINTENANCE_TIME) - && forkConfig.getLong(NEXT_MAINTENANCE_TIME) > 0) { - long nextMaintenanceTime = forkConfig.getLong(NEXT_MAINTENANCE_TIME); - dynamicPropertiesStore.put(MAINTENANCE_TIME, ByteArray.fromLong(nextMaintenanceTime)); - logger.info("The next maintenance time has been modified as {}.", - nextMaintenanceTime); - spec.commandLine().getOut().format("The next maintenance time has been modified as %d.", - nextMaintenanceTime).println(); - } - - DbTool.close(); - return 0; - } - - public static byte[] getActiveWitness(List witnesses) { - byte[] ba = new byte[witnesses.size() * Constant.ADDRESS_BYTE_ARRAY_LENGTH]; - int i = 0; - for (ByteString address : witnesses) { - System.arraycopy(address.toByteArray(), 0, - ba, i * Constant.ADDRESS_BYTE_ARRAY_LENGTH, Constant.ADDRESS_BYTE_ARRAY_LENGTH); - i++; - } - return ba; - } -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java b/tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java deleted file mode 100644 index 16dc2ff3..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java +++ /dev/null @@ -1,702 +0,0 @@ -package org.tron.plugins; - -import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; -import static org.tron.plugins.utils.Constant.ALLOW_OLD_REWARD_OPT; -import static org.tron.plugins.utils.Constant.BLOCK_INDEX_STORE; -import static org.tron.plugins.utils.Constant.BLOCK_PRODUCED_INTERVAL; -import static org.tron.plugins.utils.Constant.BLOCK_STORE; -import static org.tron.plugins.utils.Constant.CHANGE_DELEGATION; -import static org.tron.plugins.utils.Constant.CURRENT_CYCLE_NUMBER; -import static org.tron.plugins.utils.Constant.DECIMAL_OF_VI_REWARD; -import static org.tron.plugins.utils.Constant.DELEGATION_STORE; -import static org.tron.plugins.utils.Constant.DYNAMIC_PROPERTY_STORE; -import static org.tron.plugins.utils.Constant.LATEST_BLOCK_HEADER_NUMBER; -import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME_INTERVAL; -import static org.tron.plugins.utils.Constant.NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE; -import static org.tron.plugins.utils.Constant.REWARDS_KEY; -import static org.tron.plugins.utils.Constant.VOTES_ALL_WITNESSES; -import static org.tron.plugins.utils.Constant.VOTES_STORE; -import static org.tron.plugins.utils.Constant.VOTES_WITNESS_LIST; -import static org.tron.plugins.utils.Constant.WITNESS_STORE; - -import com.google.common.collect.Maps; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import java.io.File; -import java.io.IOException; -import java.math.BigInteger; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.bouncycastle.util.encoders.Hex; -import org.rocksdb.RocksDBException; -import org.tron.common.utils.ByteArray; -import org.tron.common.utils.Commons; -import org.tron.common.utils.Pair; -import org.tron.common.utils.Sha256Hash; -import org.tron.common.utils.StringUtil; -import org.tron.core.capsule.AccountCapsule; -import org.tron.core.capsule.BlockCapsule; -import org.tron.core.capsule.VotesCapsule; -import org.tron.core.capsule.WitnessCapsule; -import org.tron.core.exception.BadItemException; -import org.tron.core.store.DelegationStore; -import org.tron.plugins.utils.FileUtils; -import org.tron.plugins.utils.JsonFormat; -import org.tron.plugins.utils.db.DBInterface; -import org.tron.plugins.utils.db.DBIterator; -import org.tron.plugins.utils.db.DbTool; -import org.tron.protos.Protocol.Transaction.Contract.ContractType; -import org.tron.protos.contract.WitnessContract.VoteWitnessContract; -import picocli.CommandLine; -import picocli.CommandLine.Command; - -@Slf4j(topic = "query") -@Command(name = "query", - description = "query the latest vote and reward information from the database.", - exitCodeListHeading = "Exit Codes:%n", - exitCodeList = { - "0:Successful", - "n:Internal error: exception occurred, please check logs/toolkit.log"}) -public class DbQuery implements Callable { - - @CommandLine.Spec - CommandLine.Model.CommandSpec spec; - - @CommandLine.Option(names = {"-d", "--database-directory"}, - defaultValue = "output-directory", - description = "java-tron database directory path. Default: ${DEFAULT-VALUE}") - private String database; - - @CommandLine.Option(names = {"-c", "--config"}, - defaultValue = "query.conf", - description = "config the votes and reward query options." - + " Default: ${DEFAULT-VALUE}") - private String config; - - @CommandLine.Option(names = {"-h", "--help"}) - private boolean help; - - private DBInterface witnessStore; - private DBInterface votesStore; - private DBInterface dynamicPropertiesStore; - private DBInterface blockIndexStore; - private DBInterface blockStore; - private DBInterface accountStore; - private DBInterface delegationStore; - - boolean allWitness = false; - List witnessList = new ArrayList<>(); - private Map voters = new HashMap<>(); - private Map votesTx = new HashMap<>(); - - List rewardAddressList = new ArrayList<>(); - Map latestWitnessVi = new HashMap<>(); - - private void initStore() throws IOException, RocksDBException { - String srcDir = database + File.separator + "database"; - witnessStore = DbTool.getDB(srcDir, WITNESS_STORE); - votesStore = DbTool.getDB(srcDir, VOTES_STORE); - dynamicPropertiesStore = DbTool.getDB(srcDir, DYNAMIC_PROPERTY_STORE); - blockIndexStore = DbTool.getDB(srcDir, BLOCK_INDEX_STORE); - blockStore = DbTool.getDB(srcDir, BLOCK_STORE); - accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); - delegationStore = DbTool.getDB(srcDir, DELEGATION_STORE); - } - - - @Override - public Integer call() throws Exception { - if (help) { - spec.commandLine().usage(System.out); - return 0; - } - - Config queryConfig; - File file = Paths.get(config).toFile(); - if (file.exists() && file.isFile()) { - queryConfig = ConfigFactory.parseFile(file); - } else { - logger.error("Query config file [" + config + "] not exists!"); - spec.commandLine().getErr().format("Fork config file: %s not exists!", config).println(); - return 1; - } - - File dbFile = Paths.get(database).toFile(); - if (!dbFile.exists() || !dbFile.isDirectory()) { - logger.error("Database [" + database + "] not exists!"); - spec.commandLine().getErr().format("Database %s not exists!", database).println(); - return 1; - } - File tmp = Paths.get(database, "database", "tmp").toFile(); - if (tmp.exists()) { - FileUtils.deleteDir(tmp); - } - - initStore(); - processVotes(queryConfig); - processRewards(queryConfig); - - DbTool.close(); - return 0; - } - - private void processVotes(Config queryConfig) throws BadItemException { - if (queryConfig.hasPath(VOTES_ALL_WITNESSES)) { - allWitness = queryConfig.getBoolean(VOTES_ALL_WITNESSES); - } - if (queryConfig.hasPath(VOTES_WITNESS_LIST)) { - witnessList = queryConfig.getStringList(VOTES_WITNESS_LIST); - } - - if (!allWitness && witnessList.isEmpty()) { - spec.commandLine().getOut() - .println("skip the vote query."); - logger.info("skip the vote query."); - return; - } - - Map witnesses = new HashMap<>(); - Map oldWitnessCnt = new HashMap<>(); - DBIterator iterator = witnessStore.iterator(); - WitnessCapsule witnessCapsule; - for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { - witnessCapsule = new WitnessCapsule(iterator.getValue()); - witnesses.put(ByteString.copyFrom(iterator.getKey()), witnessCapsule); - oldWitnessCnt.put(witnessCapsule.getAddress(), witnessCapsule.getVoteCount()); - } - - Map countWitness = countVote(); - loadVotesTx(); - - AtomicInteger cnt = new AtomicInteger(); - votesTx.forEach((address, voteWitnessTx) -> { - if (allWitness || existInWitnessList(voters.get(address), witnessList)) { - cnt.getAndIncrement(); - } - }); - spec.commandLine().getOut() - .format("There are %d related system-contract vote txs", cnt.get()) - .println(); - logger.info("There are {} related system-contract vote txs", cnt.get()); - - cnt.set(-1); - votesTx.forEach((address, voteWitnessTx) -> { - VotesCapsule votesCapsule = voters.get(address); - if (!allWitness && !existInWitnessList(votesCapsule, witnessList)) { - return; - } - cnt.getAndIncrement(); - String txHashStr = voteWitnessTx.txHash.toString(); - spec.commandLine().getOut().format("tx %d: %s", cnt.get(), txHashStr).println(); - logger.info("tx {}: {}", cnt.get(), txHashStr); -// String voteWitnessStr = JsonFormat.printToString(voteWitnessTx.voteWitnessContract, true); -// spec.commandLine().getOut().println(voteWitnessStr); -// logger.info(voteWitnessStr); - String voteCapsuleStr = JsonFormat.printToString(votesCapsule.getInstance(), true); - spec.commandLine().getOut().println(voteCapsuleStr); - logger.info(voteCapsuleStr); - voters.remove(address); - }); - - cnt.set(0); - voters.forEach((address, votesCapsule) -> { - if ((allWitness || existInWitnessList(voters.get(address), witnessList)) - && votesTx.get(address) == null) { - cnt.getAndIncrement(); - } - }); - - spec.commandLine().getOut() - .format("There are %d related smart-contract votes txs", cnt.get()) - .println(); - logger.info("There are {} related smart-contract votes txs", cnt.get()); - cnt.set(-1); - voters.forEach((address, voteCapsule) -> { - if (!allWitness && !existInWitnessList(voteCapsule, witnessList)) { - return; - } - cnt.getAndIncrement(); - spec.commandLine().getOut().format("tx %d", cnt.get()).println(); - logger.info("tx {}:", cnt.get()); - String voteCapsuleStr = JsonFormat.printToString(voteCapsule.getInstance(), true); - spec.commandLine().getOut().println(voteCapsuleStr); - logger.info(voteCapsuleStr); - }); - - countWitness.forEach((address, voteCount) -> { - WitnessCapsule witness = witnesses.get(address); - if (witness == null) { - return; - } - witness.setVoteCount(witness.getVoteCount() + voteCount); - witnesses.put(address, witness); - }); - - spec.commandLine().getOut().println("Display the witness list with latest vote count: "); - logger.info("Display the witness list with latest vote count: "); - List witnessCapsuleList; - if (allWitness) { - witnessCapsuleList = new ArrayList<>(witnesses.values()); - Collections.sort(witnessCapsuleList, - Comparator.comparingLong(WitnessCapsule::getVoteCount).reversed()); - } else { - List finalWitnessCapsuleList = new ArrayList<>(); - witnessList.forEach( - witness -> { - ByteString address = ByteString.copyFrom(Commons.decodeFromBase58Check(witness)); - WitnessCapsule witnessTmp = witnesses.get(address); - if (witnessTmp == null) { - spec.commandLine().getErr().format("address: %s is not witness", witness).println(); - logger.error("address: {} is not witness", witness); - return; - } - finalWitnessCapsuleList.add(witnessTmp); - } - ); - witnessCapsuleList = finalWitnessCapsuleList; - } - -// WitnessList.Builder builder = WitnessList.newBuilder(); -// witnessCapsuleList.forEach(witness -> builder.addWitnesses(witness.getInstance())); -// String witnessesJson = JsonFormat.printToString(builder.build(), true); -// spec.commandLine().getOut().println(witnessesJson); -// logger.info(witnessesJson); - cnt.set(-1); - witnessCapsuleList.forEach( - witness -> { - cnt.getAndIncrement(); - String output = formatWitness(witness, oldWitnessCnt.get(witness.getAddress())); - spec.commandLine().getOut().println(cnt.get() + " " + output); - logger.info(cnt.get() + " " + output); - }); - } - - private String formatWitness(WitnessCapsule witnessCapsule, long previousCnt) { - return String.format("{\"address\": %s," - + "\"oldVoteCount\": %d,\"newVoteCount\": %d," - + "\"voteCountIncrement\": %d,\"url\": %s," - + "\"totalProduced\": %d,\"totalMissed\": %d,\"latestBlockNum\": %d," - + "\"latestSlotNum\": %d,\"isJobs\": %b}", - StringUtil.encode58Check(witnessCapsule.getAddress().toByteArray()), - previousCnt, - witnessCapsule.getVoteCount(), - witnessCapsule.getVoteCount() - previousCnt, - witnessCapsule.getUrl(), - witnessCapsule.getTotalProduced(), - witnessCapsule.getTotalMissed(), - witnessCapsule.getLatestBlockNum(), - witnessCapsule.getLatestSlotNum(), - witnessCapsule.getIsJobs()); - } - - - private Map countVote() { - final Map countWitness = Maps.newHashMap(); - DBIterator dbIterator = votesStore.iterator(); - long sizeCount = 0; - dbIterator.seekToFirst(); - while (dbIterator.hasNext()) { - Entry next = dbIterator.next(); - VotesCapsule votes = new VotesCapsule(next.getValue()); - voters.put(ByteString.copyFrom(next.getKey()), votes); - - votes.getOldVotes().forEach(vote -> { - ByteString voteAddress = vote.getVoteAddress(); - long voteCount = vote.getVoteCount(); - if (countWitness.containsKey(voteAddress)) { - countWitness.put(voteAddress, countWitness.get(voteAddress) - voteCount); - } else { - countWitness.put(voteAddress, -voteCount); - } - }); - votes.getNewVotes().forEach(vote -> { - ByteString voteAddress = vote.getVoteAddress(); - long voteCount = vote.getVoteCount(); - if (countWitness.containsKey(voteAddress)) { - countWitness.put(voteAddress, countWitness.get(voteAddress) + voteCount); - } else { - countWitness.put(voteAddress, voteCount); - } - }); - sizeCount++; - } - spec.commandLine().getOut().format("There are total %d new votes in this epoch", sizeCount) - .println(); - logger.info("There are total {} new votes in this epoch", sizeCount); - return countWitness; - } - - private void loadVotesTx() throws BadItemException { - long maintenanceTimeInterval = ByteArray - .toLong(dynamicPropertiesStore.get(MAINTENANCE_TIME_INTERVAL)); - long maintenanceBlockCnt = maintenanceTimeInterval / BLOCK_PRODUCED_INTERVAL; - long latestBlockNumber = ByteArray - .toLong(dynamicPropertiesStore.get(LATEST_BLOCK_HEADER_NUMBER)); - long startBlock = - latestBlockNumber > maintenanceBlockCnt ? latestBlockNumber - maintenanceBlockCnt : 0; - long block; - for (block = startBlock; block <= latestBlockNumber; block++) { - byte[] blockHash = blockIndexStore.get(ByteArray.fromLong(block)); - BlockCapsule blockCapsule = new BlockCapsule(blockStore.get(blockHash)); - - blockCapsule.getTransactions().forEach(txCapsule -> { - ContractType txType = txCapsule.getInstance().getRawData().getContract(0).getType(); - if (!txType.equals(ContractType.VoteWitnessContract)) { - return; - } - try { - VoteWitnessContract voteWitnessContract = txCapsule.getInstance().getRawData() - .getContract(0) - .getParameter().unpack(VoteWitnessContract.class); - ByteString ownerAddress = voteWitnessContract.getOwnerAddress(); - voteWitnessContract.getVotesList().forEach(vote -> { - if (voters.keySet().contains(ownerAddress)) { - votesTx.put(ownerAddress, - new VoteWitnessTx(txCapsule.getTransactionId(), voteWitnessContract)); - } - }); - } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); - System.exit(-1); - } - }); - } - } - - private class VoteWitnessTx { - - private Sha256Hash txHash; - private VoteWitnessContract voteWitnessContract; - - VoteWitnessTx(Sha256Hash txHash, VoteWitnessContract voteWitnessContract) { - this.txHash = txHash; - this.voteWitnessContract = voteWitnessContract; - } - - private Boolean existInWitnessList(List witnessList) { - AtomicBoolean exist = new AtomicBoolean(false); - voteWitnessContract.getVotesList().forEach(vote -> { - if (witnessList.contains(StringUtil.encode58Check(vote.getVoteAddress().toByteArray()))) { - exist.set(true); - } - }); - return exist.get(); - } - } - - private Boolean existInWitnessList(VotesCapsule votesCapsule, List witnessList) { - AtomicBoolean exist = new AtomicBoolean(false); - votesCapsule.getOldVotes().forEach(vote -> { - if (witnessList.contains(StringUtil.encode58Check(vote.getVoteAddress().toByteArray()))) { - exist.set(true); - } - }); - votesCapsule.getNewVotes().forEach(vote -> { - if (witnessList.contains(StringUtil.encode58Check(vote.getVoteAddress().toByteArray()))) { - exist.set(true); - } - }); - return exist.get(); - } - - private void processRewards(Config config) { - if (config.hasPath(REWARDS_KEY)) { - rewardAddressList = config.getStringList(REWARDS_KEY); - } - if (rewardAddressList.isEmpty()) { - spec.commandLine().getOut() - .println("skip the reward query."); - logger.info("skip the reward query."); - return; - } - spec.commandLine().getOut().println("\nBegin to query the reward"); - - loadAccumulateWitnessVi(); - rewardAddressList.forEach(address -> { - long reward = queryReward(Commons.decodeFromBase58Check(address), false); - long latestReward = queryReward(Commons.decodeFromBase58Check(address), true); - spec.commandLine().getOut() - .format("address: %s, cycle reward: %d, latest reward: %d, increment: %d", address, - reward, latestReward, (latestReward - reward)).println(); - logger.info("address: {}, cycle reward: {}, latest reward: {}, increment: {}", address, - reward, latestReward, (latestReward - reward)); - }); - - spec.commandLine().getOut().println("Finish querying the reward"); - } - - private void loadAccumulateWitnessVi() { - long cycle = getCurrentCycle(); - DBIterator iterator = witnessStore.iterator(); - WitnessCapsule witnessCapsule; - for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { - witnessCapsule = new WitnessCapsule(iterator.getValue()); - accumulateWitnessVi(cycle, witnessCapsule.createDbKey(), witnessCapsule.getVoteCount()); - } - } - - private void accumulateWitnessVi(long cycle, byte[] address, long voteCount) { - BigInteger preVi = getWitnessVi(cycle - 1, address); - long reward = getReward(cycle, address); - if (reward == 0 || voteCount == 0) { // Just forward pre vi - if (!BigInteger.ZERO.equals(preVi)) { // Zero vi will not be record - setWitnessVi(cycle, address, preVi); - } - } else { // Accumulate delta vi - BigInteger deltaVi = BigInteger.valueOf(reward) - .multiply(DECIMAL_OF_VI_REWARD) - .divide(BigInteger.valueOf(voteCount)); - setWitnessVi(cycle, address, preVi.add(deltaVi)); - } - } - - private void setWitnessVi(long cycle, byte[] address, BigInteger value) { - byte[] key = buildViKey(cycle, address); - latestWitnessVi.put(ByteArray.toHexString(key), value); - } - - private BigInteger getWitnessViFromMap(long cycle, byte[] address) { - byte[] key = buildViKey(cycle, address); - BigInteger value = latestWitnessVi.get(ByteArray.toHexString(key)); - if (value == null) { - return BigInteger.ZERO; - } else { - return value; - } - } - - private long queryReward(byte[] address, boolean isLatest) { - if (ByteArray.toLong(dynamicPropertiesStore.get(CHANGE_DELEGATION)) != 1) { - return 0; - } - - byte[] accountValue = accountStore.get(address); - if (accountValue == null) { - return 0; - } - AccountCapsule accountCapsule = new AccountCapsule(accountValue); - long beginCycle = getBeginCycle(address); - long endCycle = getEndCycle(address); - long currentCycle = getCurrentCycle(); - - long reward = 0; - if (beginCycle > currentCycle) { - return accountCapsule.getAllowance(); - } - - //withdraw the latest cycle reward - if (beginCycle + 1 == endCycle && beginCycle < currentCycle) { - AccountCapsule account = getAccountVote(beginCycle, address); - if (account != null) { - reward = computeReward(beginCycle, endCycle, account, false); - } - beginCycle += 1; - } - - endCycle = currentCycle; - if (CollectionUtils.isEmpty(accountCapsule.getVotesList())) { - return reward + accountCapsule.getAllowance(); - } - if ((!isLatest && beginCycle < endCycle) || (isLatest && beginCycle <= endCycle)) { - reward += computeReward(beginCycle, endCycle, accountCapsule, isLatest); - } - return reward + accountCapsule.getAllowance(); - } - - private long getBeginCycle(byte[] address) { - byte[] beginCycleValue = delegationStore.get(address); - return beginCycleValue == null ? 0 : ByteArray.toLong(beginCycleValue); - } - - private long getEndCycle(byte[] address) { - byte[] endCycleValue = delegationStore.get(buildEndCycleKey(address)); - return endCycleValue == null ? -1L : ByteArray.toLong(endCycleValue); - } - - private byte[] buildEndCycleKey(byte[] address) { - return ("end-" + Hex.toHexString(address)).getBytes(); - } - - private long getCurrentCycle() { - byte[] currentCycleValue = dynamicPropertiesStore.get(CURRENT_CYCLE_NUMBER); - return currentCycleValue == null ? 0L : ByteArray.toLong(currentCycleValue); - } - - private AccountCapsule getAccountVote(long cycle, byte[] address) { - byte[] value = delegationStore.get(buildAccountVoteKey(cycle, address)); - if (value == null) { - return null; - } else { - return new AccountCapsule(value); - } - } - - private byte[] buildAccountVoteKey(long cycle, byte[] address) { - return (cycle + "-" + Hex.toHexString(address) + "-account-vote").getBytes(); - } - - private long getNewRewardAlgorithmEffectiveCycle() { - byte[] value = dynamicPropertiesStore.get(NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE); - return value == null ? 0L : ByteArray.toLong(value); - } - - private Boolean allowOldRewardOpt() { - byte[] value = dynamicPropertiesStore.get(ALLOW_OLD_REWARD_OPT); - return value != null && ByteArray.toLong(value) == 1; - } - - private BigInteger getWitnessVi(long cycle, byte[] address) { - byte[] value = delegationStore.get(buildViKey(cycle, address)); - if (value == null) { - return BigInteger.ZERO; - } else { - return new BigInteger(value); - } - } - - private byte[] buildViKey(long cycle, byte[] address) { - return (cycle + "-" + Hex.toHexString(address) + "-vi").getBytes(); - } - - public long getReward(long cycle, byte[] address) { - byte[] value = delegationStore.get(buildRewardKey(cycle, address)); - if (value == null) { - return 0L; - } else { - return ByteArray.toLong(value); - } - } - - private byte[] buildRewardKey(long cycle, byte[] address) { - return (cycle + "-" + Hex.toHexString(address) + "-reward").getBytes(); - } - - public long getWitnessVote(long cycle, byte[] address) { - byte[] value = delegationStore.get(buildVoteKey(cycle, address)); - if (value == null) { - return -1L; - } else { - return ByteArray.toLong(value); - } - } - - private byte[] buildVoteKey(long cycle, byte[] address) { - return (cycle + "-" + Hex.toHexString(address) + "-vote").getBytes(); - } - - - /** - * Compute reward from begin cycle to end cycle, which endCycle must greater than beginCycle. - * While computing reward after new reward algorithm taking effective cycle number, it will use - * new algorithm instead of old way. - * - * @param beginCycle begin cycle (include) - * @param endCycle end cycle (exclude) - * @param accountCapsule account capsule - * @param isLatest whether to compute the latest reward - * @return total reward - */ - private long computeReward(long beginCycle, long endCycle, AccountCapsule accountCapsule, - boolean isLatest) { - if ((!isLatest && beginCycle >= endCycle) || (isLatest && beginCycle > endCycle)) { - return 0; - } - - long reward = 0; - long newAlgorithmCycle = getNewRewardAlgorithmEffectiveCycle(); - List> srAddresses = accountCapsule.getVotesList().stream() - .map(vote -> new Pair<>(vote.getVoteAddress().toByteArray(), vote.getVoteCount())) - .collect(Collectors.toList()); - if (beginCycle < newAlgorithmCycle) { - long oldEndCycle = Math.min(endCycle, newAlgorithmCycle); - reward = getOldReward(beginCycle, oldEndCycle, srAddresses); - beginCycle = oldEndCycle; - } - if ((!isLatest && beginCycle < endCycle) || (isLatest && beginCycle <= endCycle)) { - for (Pair vote : srAddresses) { - byte[] srAddress = vote.getKey(); - BigInteger beginVi = getWitnessVi(beginCycle - 1, srAddress); - BigInteger endVi; - if (!isLatest) { - endVi = getWitnessVi(endCycle - 1, srAddress); - } else { - endCycle = getCurrentCycle(); - endVi = getWitnessViFromMap(endCycle, srAddress); - } - BigInteger deltaVi = endVi.subtract(beginVi); - if (deltaVi.signum() <= 0) { - continue; - } - long userVote = vote.getValue(); - reward += deltaVi.multiply(BigInteger.valueOf(userVote)) - .divide(DECIMAL_OF_VI_REWARD).longValue(); - } - } - return reward; - } - - private long getNewRewardAlgorithmReward(long beginCycle, long endCycle, - List> votes) { - long reward = 0; - if (beginCycle < endCycle) { - for (Pair vote : votes) { - byte[] srAddress = vote.getKey(); - BigInteger beginVi = getWitnessVi(beginCycle - 1, srAddress); - BigInteger endVi = getWitnessVi(endCycle - 1, srAddress); - BigInteger deltaVi = endVi.subtract(beginVi); - if (deltaVi.signum() <= 0) { - continue; - } - long userVote = vote.getValue(); - reward += deltaVi.multiply(BigInteger.valueOf(userVote)) - .divide(DECIMAL_OF_VI_REWARD).longValue(); - } - } - return reward; - } - - private long computeReward(long cycle, List> votes) { - long reward = 0; - for (Pair vote : votes) { - byte[] srAddress = vote.getKey(); - long totalReward = getReward(cycle, srAddress); - if (totalReward <= 0) { - continue; - } - long totalVote = getWitnessVote(cycle, srAddress); - if (totalVote == DelegationStore.REMARK || totalVote == 0) { - continue; - } - long userVote = vote.getValue(); - // Replace floating-point division with integer-based calculation - reward += (userVote * totalReward) / totalVote; - } - return reward; - } - - private long getOldReward(long begin, long end, List> votes) { - if (allowOldRewardOpt()) { - return getNewRewardAlgorithmReward(begin, end, votes); - } - long reward = 0; - for (long cycle = begin; cycle < end; cycle++) { - reward += computeReward(cycle, votes); - } - return reward; - } - -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java b/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java deleted file mode 100644 index 9cb12b49..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.tron.plugins.comparator; - -import org.iq80.leveldb.DBComparator; -import org.tron.plugins.utils.MarketUtils; - -public class MarketOrderPriceComparatorForLevelDB implements DBComparator { - - @Override - public String name() { - return "MarketOrderPriceComparator"; - } - - @Override - public byte[] findShortestSeparator(byte[] start, byte[] limit) { - return new byte[0]; - } - - @Override - public byte[] findShortSuccessor(byte[] key) { - return new byte[0]; - } - - @Override - public int compare(byte[] o1, byte[] o2) { - return MarketUtils.comparePriceKey(o1, o2); - } - -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java b/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java deleted file mode 100644 index 40d65516..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.tron.plugins.comparator; - -import org.rocksdb.ComparatorOptions; -import org.rocksdb.DirectSlice; -import org.rocksdb.util.DirectBytewiseComparator; -import org.tron.plugins.utils.MarketUtils; - -public class MarketOrderPriceComparatorForRockDB extends DirectBytewiseComparator { - - public MarketOrderPriceComparatorForRockDB(final ComparatorOptions copt) { - super(copt); - } - - @Override - public String name() { - return "MarketOrderPriceComparator"; - } - - @Override - public int compare(final DirectSlice a, final DirectSlice b) { - return MarketUtils.comparePriceKey(convertDataToBytes(a), convertDataToBytes(b)); - } - - /** - * DirectSlice.data().array will throw UnsupportedOperationException. - * */ - public byte[] convertDataToBytes(DirectSlice directSlice) { - int capacity = directSlice.data().capacity(); - byte[] bytes = new byte[capacity]; - - for (int i = 0; i < capacity; i++) { - bytes[i] = directSlice.get(i); - } - - return bytes; - } - -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java deleted file mode 100644 index 7fcc380c..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.tron.plugins.utils; - -import java.math.BigInteger; - -public class Constant { - - public static final String WITNESS_KEY = "witnesses"; - public static final String WITNESS_ADDRESS = "address"; - public static final String WITNESS_URL = "url"; - public static final String WITNESS_VOTE = "voteCount"; - public static final String ACCOUNTS_KEY = "accounts"; - public static final String ACCOUNT_NAME = "accountName"; - public static final String ACCOUNT_TYPE = "accountType"; - public static final String ACCOUNT_ADDRESS = "address"; - public static final String ACCOUNT_BALANCE = "balance"; - public static final String ACCOUNT_OWNER = "owner"; - public static final String ACCOUNT_TRC10_ID = "trc10Id"; - public static final String ACCOUNT_TRC10_BALANCE = "trc10Balance"; - - public static final String TRC20_CONTRACTS_KEY = "trc20Contracts"; - public static final String TRC20_CONTRACT_ADDRESS = "contractAddress"; - public static final String TRC20_BALANCES_POSITION = "balancesSlotPosition"; - public static final String TRC20_ACCOUNT = "address"; - public static final String TRC20_BALANCE = "balance"; - - public static final String LATEST_BLOCK_TIMESTAMP = "latestBlockHeaderTimestamp"; - public static final String MAINTENANCE_INTERVAL = "maintenanceTimeInterval"; - public static final String NEXT_MAINTENANCE_TIME = "nextMaintenanceTime"; - public static final int MAX_ACTIVE_WITNESS_NUM = 27; - - public static final String WITNESS_STORE = "witness"; - public static final String WITNESS_SCHEDULE_STORE = "witness_schedule"; - public static final String ACCOUNT_STORE = "account"; - public static final String DYNAMIC_PROPERTY_STORE = "properties"; - public static final String ASSET_ISSUE_V2 = "asset-issue-v2"; - public static final String ACCOUNT_ASSET = "account-asset"; - public static final String CONTRACT_STORE = "contract"; - public static final String STORAGE_ROW_STORE = "storage-row"; - public static final String VOTES_STORE = "votes"; - public static final String BLOCK_INDEX_STORE = "block-index"; - public static final String BLOCK_STORE = "block"; - public static final String DELEGATION_STORE = "delegation"; - - public static final byte[] LATEST_BLOCK_HEADER_TIMESTAMP = "latest_block_header_timestamp" - .getBytes(); - public static final byte[] MAINTENANCE_TIME_INTERVAL = "MAINTENANCE_TIME_INTERVAL".getBytes(); - public static final byte[] LATEST_BLOCK_HEADER_NUMBER = "latest_block_header_number".getBytes(); - public static final byte[] MAINTENANCE_TIME = "NEXT_MAINTENANCE_TIME".getBytes(); - public static final byte[] ACTIVE_WITNESSES = "active_witnesses".getBytes(); - public static final int ADDRESS_BYTE_ARRAY_LENGTH = 21; - - public static final String VOTES_ALL_WITNESSES = "vote.allWitnesses"; - public static final String VOTES_WITNESS_LIST = "vote.witnessList"; - public static final int BLOCK_PRODUCED_INTERVAL = 3000; - - public static final String REWARDS_KEY = "reward"; - public static final byte[] CHANGE_DELEGATION = "CHANGE_DELEGATION".getBytes(); - public static final byte[] CURRENT_CYCLE_NUMBER = "CURRENT_CYCLE_NUMBER".getBytes(); - public static final byte[] NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE = - "NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE".getBytes(); - public static final byte[] ALLOW_OLD_REWARD_OPT = "ALLOW_OLD_REWARD_OPT".getBytes(); - public static final BigInteger DECIMAL_OF_VI_REWARD = BigInteger.valueOf(10).pow(18); -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/HttpSelfFormatFieldName.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/HttpSelfFormatFieldName.java deleted file mode 100644 index c40f10da..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/HttpSelfFormatFieldName.java +++ /dev/null @@ -1,283 +0,0 @@ -package org.tron.plugins.utils; - -import java.util.HashMap; -import java.util.Map; - -public class HttpSelfFormatFieldName { - - private static Map AddressFieldNameMap = new HashMap<>(); - private static Map NameFieldNameMap = new HashMap<>(); - - static { - //***** api.proto ***** - //DelegatedResourceMessage - AddressFieldNameMap.put("protocol.DelegatedResourceMessage.fromAddress", 1); - AddressFieldNameMap.put("protocol.DelegatedResourceMessage.toAddress", 1); - //TransactionSignWeight - AddressFieldNameMap.put("protocol.TransactionSignWeight.approved_list", 1); - //TransactionApprovedList - AddressFieldNameMap.put("protocol.TransactionApprovedList.approved_list", 1); - //PrivateParameters - AddressFieldNameMap.put("protocol.PrivateParameters.transparent_from_address", 1); - AddressFieldNameMap.put("protocol.PrivateParameters.transparent_to_address", 1); - //PrivateParametersWithoutAsk - AddressFieldNameMap.put("protocol.PrivateParametersWithoutAsk.transparent_from_address", 1); - AddressFieldNameMap.put("protocol.PrivateParametersWithoutAsk.transparent_to_address", 1); - //PrivateShieldedTRC20Parameters - AddressFieldNameMap.put( - "protocol.PrivateShieldedTRC20Parameters.transparent_to_address", 1); - AddressFieldNameMap.put( - "protocol.PrivateShieldedTRC20Parameters.shielded_TRC20_contract_address", 1); - //PrivateShieldedTRC20ParametersWithoutAsk - AddressFieldNameMap.put( - "protocol.PrivateShieldedTRC20ParametersWithoutAsk.transparent_to_address", 1); - AddressFieldNameMap.put( - "protocol.PrivateShieldedTRC20ParametersWithoutAsk.shielded_TRC20_contract_address", 1); - //IvkDecryptTRC20Parameters - AddressFieldNameMap.put( - "protocol.IvkDecryptTRC20Parameters.shielded_TRC20_contract_address", 1); - //OvkDecryptTRC20Parameters - AddressFieldNameMap.put( - "protocol.OvkDecryptTRC20Parameters.shielded_TRC20_contract_address", 1); - //NfTRC20Parameters - AddressFieldNameMap.put( - "protocol.NfTRC20Parameters.shielded_TRC20_contract_address", 1); - //ShieldedTRC20TriggerContractParameters - AddressFieldNameMap.put( - "protocol.ShieldedTRC20TriggerContractParameters.transparent_to_address", 1); - AddressFieldNameMap.put( - "protocol.DecryptNotesTRC20.NoteTx.transparent_to_address", 1); - - //***** Contract.proto ***** - //AccountCreateContract - AddressFieldNameMap.put("protocol.AccountCreateContract.owner_address", 1); - AddressFieldNameMap.put("protocol.AccountCreateContract.account_address", 1); - //AccountUpdateContract - AddressFieldNameMap.put("protocol.AccountUpdateContract.owner_address", 1); - //SetAccountIdContract - AddressFieldNameMap.put("protocol.SetAccountIdContract.owner_address", 1); - //TransferContract - AddressFieldNameMap.put("protocol.TransferContract.owner_address", 1); - AddressFieldNameMap.put("protocol.TransferContract.to_address", 1); - //CancelDeferredTransactionContract - AddressFieldNameMap.put("protocol.CancelDeferredTransactionContract.ownerAddress", 1); - //TransferAssetContract - AddressFieldNameMap.put("protocol.TransferAssetContract.owner_address", 1); - AddressFieldNameMap.put("protocol.TransferAssetContract.to_address", 1); - //VoteAssetContract - AddressFieldNameMap.put("protocol.VoteAssetContract.owner_address", 1); - AddressFieldNameMap.put("protocol.VoteAssetContract.vote_address", 1); - //VoteWitnessContract - AddressFieldNameMap.put("protocol.VoteWitnessContract.Vote.vote_address", 1); - AddressFieldNameMap.put("protocol.VoteWitnessContract.owner_address", 1); - //UpdateSettingContract - AddressFieldNameMap.put("protocol.UpdateSettingContract.owner_address", 1); - AddressFieldNameMap.put("protocol.UpdateSettingContract.contract_address", 1); - //UpdateEnergyLimitContract - AddressFieldNameMap.put("protocol.UpdateEnergyLimitContract.owner_address", 1); - AddressFieldNameMap.put("protocol.UpdateEnergyLimitContract.contract_address", 1); - //ClearABIContract - AddressFieldNameMap.put("protocol.ClearABIContract.owner_address", 1); - AddressFieldNameMap.put("protocol.ClearABIContract.contract_address", 1); - //WitnessCreateContract - AddressFieldNameMap.put("protocol.WitnessCreateContract.owner_address", 1); - //WitnessUpdateContract - AddressFieldNameMap.put("protocol.WitnessUpdateContract.owner_address", 1); - //AssetIssueContract - AddressFieldNameMap.put("protocol.AssetIssueContract.owner_address", 1); - //ParticipateAssetIssueContract - AddressFieldNameMap.put("protocol.ParticipateAssetIssueContract.owner_address", 1); - AddressFieldNameMap.put("protocol.ParticipateAssetIssueContract.to_address", 1); - //FreezeBalanceContract - AddressFieldNameMap.put("protocol.FreezeBalanceContract.owner_address", 1); - AddressFieldNameMap.put("protocol.FreezeBalanceContract.receiver_address", 1); - //UnfreezeBalanceContract - AddressFieldNameMap.put("protocol.UnfreezeBalanceContract.owner_address", 1); - AddressFieldNameMap.put("protocol.UnfreezeBalanceContract.receiver_address", 1); - //UnfreezeAssetContract - AddressFieldNameMap.put("protocol.UnfreezeAssetContract.owner_address", 1); - //WithdrawBalanceContract - AddressFieldNameMap.put("protocol.WithdrawBalanceContract.owner_address", 1); - //UpdateAssetContract - AddressFieldNameMap.put("protocol.UpdateAssetContract.owner_address", 1); - //ProposalCreateContract - AddressFieldNameMap.put("protocol.ProposalCreateContract.owner_address", 1); - //ProposalApproveContract - AddressFieldNameMap.put("protocol.ProposalApproveContract.owner_address", 1); - //ProposalDeleteContract - AddressFieldNameMap.put("protocol.ProposalDeleteContract.owner_address", 1); - //CreateSmartContract - AddressFieldNameMap.put("protocol.CreateSmartContract.owner_address", 1); - //TriggerSmartContract - AddressFieldNameMap.put("protocol.TriggerSmartContract.owner_address", 1); - AddressFieldNameMap.put("protocol.TriggerSmartContract.contract_address", 1); - //BuyStorageContract - AddressFieldNameMap.put("protocol.BuyStorageContract.owner_address", 1); - //BuyStorageBytesContract - AddressFieldNameMap.put("protocol.BuyStorageBytesContract.owner_address", 1); - //SellStorageContract - AddressFieldNameMap.put("protocol.SellStorageContract.owner_address", 1); - //ExchangeCreateContract - AddressFieldNameMap.put("protocol.ExchangeCreateContract.owner_address", 1); - //ExchangeInjectContract - AddressFieldNameMap.put("protocol.ExchangeInjectContract.owner_address", 1); - //ExchangeWithdrawContract - AddressFieldNameMap.put("protocol.ExchangeWithdrawContract.owner_address", 1); - //ExchangeTransactionContract - AddressFieldNameMap.put("protocol.ExchangeTransactionContract.owner_address", 1); - //AccountPermissionUpdateContract - AddressFieldNameMap.put("protocol.AccountPermissionUpdateContract.owner_address", 1); - //UpdateBrokerageContract - AddressFieldNameMap.put("protocol.UpdateBrokerageContract.owner_address", 1); - //ShieldedTransferContract - AddressFieldNameMap.put("protocol.ShieldedTransferContract.transparent_from_address", 1); - AddressFieldNameMap.put("protocol.ShieldedTransferContract.transparent_to_address", 1); - //FreezeBalanceV2Contract - AddressFieldNameMap.put("protocol.FreezeBalanceV2Contract.owner_address", 1); - //UnfreezeBalanceV2Contract - AddressFieldNameMap.put("protocol.UnfreezeBalanceV2Contract.owner_address", 1); - //WithdrawExpireUnfreezeContract - AddressFieldNameMap.put("protocol.WithdrawExpireUnfreezeContract.owner_address", 1); - //DelegateResourceContract - AddressFieldNameMap.put("protocol.DelegateResourceContract.owner_address", 1); - AddressFieldNameMap.put("protocol.DelegateResourceContract.receiver_address", 1); - //UnDelegateResourceContract - AddressFieldNameMap.put("protocol.UnDelegateResourceContract.owner_address", 1); - AddressFieldNameMap.put("protocol.UnDelegateResourceContract.receiver_address", 1); - //CancelAllUnfreezeV2Contract - AddressFieldNameMap.put("protocol.CancelAllUnfreezeV2Contract.owner_address", 1); - AddressFieldNameMap.put("protocol.CanDelegatedMaxSizeRequestMessage.owner_address", 1); - AddressFieldNameMap.put("protocol.GetAvailableUnfreezeCountRequestMessage.owner_address", 1); - AddressFieldNameMap.put("protocol.CanWithdrawUnfreezeAmountRequestMessage.owner_address", 1); - - //***** Tron.proto ***** - //AccountId - AddressFieldNameMap.put("protocol.AccountId.address", 1); - //Vote - AddressFieldNameMap.put("protocol.Vote.vote_address", 1); - //Proposal - AddressFieldNameMap.put("protocol.Proposal.proposer_address", 1); - AddressFieldNameMap.put("protocol.Proposal.approvals", 1); - //Exchange - AddressFieldNameMap.put("protocol.Exchange.creator_address", 1); - //Account - AddressFieldNameMap.put("protocol.Account.address", 1); - //Key - AddressFieldNameMap.put("protocol.Key.address", 1); - //DelegatedResource - AddressFieldNameMap.put("protocol.DelegatedResource.from", 1); - AddressFieldNameMap.put("protocol.DelegatedResource.to", 1); - //Witness - AddressFieldNameMap.put("protocol.Witness.address", 1); - //Votes - AddressFieldNameMap.put("protocol.Votes.address", 1); - //TransactionInfo - AddressFieldNameMap.put("protocol.TransactionInfo.Log.address", 1); - AddressFieldNameMap.put("protocol.TransactionInfo.contract_address", 1); - //DeferredTransaction - AddressFieldNameMap.put("protocol.DeferredTransaction.senderAddress", 1); - AddressFieldNameMap.put("protocol.DeferredTransaction.receiverAddress", 1); - //BlockHeader - AddressFieldNameMap.put("protocol.BlockHeader.raw.witness_address", 1); - //SmartContract - AddressFieldNameMap.put("protocol.SmartContract.origin_address", 1); - AddressFieldNameMap.put("protocol.SmartContract.contract_address", 1); - //InternalTransaction - AddressFieldNameMap.put("protocol.InternalTransaction.caller_address", 1); - AddressFieldNameMap.put("protocol.InternalTransaction.transferTo_address", 1); - //DelegatedResourceAccountIndex - AddressFieldNameMap.put("protocol.DelegatedResourceAccountIndex.account", 1); - AddressFieldNameMap.put("protocol.DelegatedResourceAccountIndex.fromAccounts", 1); - AddressFieldNameMap.put("protocol.DelegatedResourceAccountIndex.toAccounts", 1); - - AddressFieldNameMap.put("protocol.AccountIdentifier.address", 1); - AddressFieldNameMap.put("protocol.TransactionBalanceTrace.Operation.address", 1); - - //***** api.proto ***** - //Return - NameFieldNameMap.put("protocol.Return.message", 1); - //Address - NameFieldNameMap.put("protocol.Address.host", 1); - //Note - NameFieldNameMap.put("protocol.Note.memo", 1); - - //***** Contract.proto ***** - //AccountUpdateContract - NameFieldNameMap.put("protocol.AccountUpdateContract.account_name", 1); - //SetAccountIdContract - NameFieldNameMap.put("protocol.SetAccountIdContract.account_id", 1); - //TransferAssetContract - NameFieldNameMap.put("protocol.TransferAssetContract.asset_name", 1); - //WitnessCreateContract - NameFieldNameMap.put("protocol.WitnessCreateContract.url", 1); - //WitnessUpdateContract - NameFieldNameMap.put("protocol.WitnessUpdateContract.update_url", 1); - //AssetIssueContract - NameFieldNameMap.put("protocol.AssetIssueContract.name", 1); - NameFieldNameMap.put("protocol.AssetIssueContract.abbr", 1); - NameFieldNameMap.put("protocol.AssetIssueContract.description", 1); - NameFieldNameMap.put("protocol.AssetIssueContract.url", 1); - //ParticipateAssetIssueContract - NameFieldNameMap.put("protocol.ParticipateAssetIssueContract.asset_name", 1); - //UpdateAssetContract - NameFieldNameMap.put("protocol.UpdateAssetContract.description", 1); - NameFieldNameMap.put("protocol.UpdateAssetContract.url", 1); - //ExchangeCreateContract - NameFieldNameMap.put("protocol.ExchangeCreateContract.first_token_id", 1); - NameFieldNameMap.put("protocol.ExchangeCreateContract.second_token_id", 1); - //ExchangeInjectContract - NameFieldNameMap.put("protocol.ExchangeInjectContract.token_id", 1); - //ExchangeWithdrawContract - NameFieldNameMap.put("protocol.ExchangeWithdrawContract.token_id", 1); - //ExchangeTransactionContract - NameFieldNameMap.put("protocol.ExchangeTransactionContract.token_id", 1); - - //***** Tron.proto ***** - //AccountId - NameFieldNameMap.put("protocol.AccountId.name", 1); - //Exchange - NameFieldNameMap.put("protocol.Exchange.first_token_id", 1); - NameFieldNameMap.put("protocol.Exchange.second_token_id", 1); - //Account - NameFieldNameMap.put("protocol.Account.account_name", 1); - NameFieldNameMap.put("protocol.Account.asset_issued_name", 1); - NameFieldNameMap.put("protocol.Account.asset_issued_ID", 1); - NameFieldNameMap.put("protocol.Account.account_id", 1); - //authority - NameFieldNameMap.put("protocol.authority.permission_name", 1); - //Transaction - NameFieldNameMap.put("protocol.Transaction.Contract.ContractName", 1); - //TransactionInfo - NameFieldNameMap.put("protocol.TransactionInfo.resMessage", 1); - - //***** market.proto ***** - // MarketSellAssetContract - AddressFieldNameMap.put("protocol.MarketSellAssetContract.owner_address", 1); - NameFieldNameMap.put("protocol.MarketSellAssetContract.sell_token_id", 1); - NameFieldNameMap.put("protocol.MarketSellAssetContract.buy_token_id", 1); - - // MarketCancelOrderContract - AddressFieldNameMap.put("protocol.MarketCancelOrderContract.owner_address", 1); - - // MarketOrder - AddressFieldNameMap.put("protocol.MarketOrder.owner_address", 1); - NameFieldNameMap.put("protocol.MarketOrder.sell_token_id", 1); - NameFieldNameMap.put("protocol.MarketOrder.buy_token_id", 1); - - // MarketOrderPair - NameFieldNameMap.put("protocol.MarketOrderPair.sell_token_id", 1); - NameFieldNameMap.put("protocol.MarketOrderPair.buy_token_id", 1); - - // MarketPriceList - NameFieldNameMap.put("protocol.MarketPriceList.sell_token_id", 1); - NameFieldNameMap.put("protocol.MarketPriceList.buy_token_id", 1); - } - - public static boolean isAddressFormat(final String name) { - return AddressFieldNameMap.containsKey(name); - } - - public static boolean isNameStringFormat(final String name) { - return NameFieldNameMap.containsKey(name); - } -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/JsonFormat.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/JsonFormat.java deleted file mode 100644 index 1ab5c07c..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/JsonFormat.java +++ /dev/null @@ -1,1663 +0,0 @@ -package org.tron.plugins.utils; - -/* - Copyright (c) 2009, Orbitz World Wide - All rights reserved. - - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of the Orbitz World Wide nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -import com.alibaba.fastjson.JSON; -import com.google.common.collect.ImmutableSet; -import com.google.protobuf.ByteString; -import com.google.protobuf.Descriptors; -import com.google.protobuf.Descriptors.Descriptor; -import com.google.protobuf.Descriptors.EnumDescriptor; -import com.google.protobuf.Descriptors.EnumValueDescriptor; -import com.google.protobuf.Descriptors.FieldDescriptor; -import com.google.protobuf.ExtensionRegistry; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; -import com.google.protobuf.UnknownFieldSet; -import java.io.IOException; -import java.math.BigInteger; -import java.nio.CharBuffer; -import java.text.CharacterIterator; -import java.text.StringCharacterIterator; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.apache.commons.lang3.StringUtils; -import org.tron.common.utils.ByteArray; -import org.tron.common.utils.Commons; -import org.tron.common.utils.StringUtil; -import org.tron.protos.contract.BalanceContract; - -/** - * Provide ascii text parsing and formatting support for proto2 instances. The implementation - * largely follows google/protobuf/text_format.cc. - * - *

(c) 2009-10 Orbitz World Wide. All Rights Reserved. - * - * @author eliran.bivas@gmail.com Eliran Bivas - * @author aantonov@orbitz.com Alex Antonov

* Based on the original code by: - * @author wenboz@google.com Wenbo Zhu - * @author kenton@google.com Kenton Varda - */ -public class JsonFormat { - - private static final int BUFFER_SIZE = 4096; - private static final Pattern DIGITS = Pattern.compile( - "[0-9]", - Pattern.CASE_INSENSITIVE); - private static final String WRITING_STRING_BUILDER_EXCEPTION - = "Writing to a StringBuilder threw an IOException (should never happen)."; - private static final String EXPECTED_STRING = "Expected string."; - private static final String MISSING_END_QUOTE = "String missing ending quote."; - - public static final boolean ALWAYS_OUTPUT_DEFAULT_VALUE_FIELDS = true; - public static final Set> MESSAGES = ImmutableSet.of( - BalanceContract.AccountBalanceResponse.class, - BalanceContract.BlockBalanceTrace.class, - BalanceContract.TransactionBalanceTrace.Operation.class, - BalanceContract.TransactionBalanceTrace.class - ); - - /** - * Outputs a textual representation of the Protocol Message supplied into the parameter output. - * (This representation is the new version of the classic "ProtocolPrinter" output from the - * original Protocol Buffer system) - */ - public static void print(Message message, Appendable output, boolean selfType) - throws IOException { - JsonGenerator generator = new JsonGenerator(output); - generator.print("{"); - print(message, generator, selfType); - generator.print("}"); - } - - /** - * Outputs a textual representation of {@code fields} to {@code output}. - */ - public static void print(UnknownFieldSet fields, Appendable output, boolean selfType) - throws IOException { - JsonGenerator generator = new JsonGenerator(output); - generator.print("{"); - printUnknownFields(fields, generator, selfType); - generator.print("}"); - } - - protected static void print(Message message, JsonGenerator generator, boolean selfType) - throws IOException { - Map fieldsToPrint = new TreeMap<>(message.getAllFields()); - if (ALWAYS_OUTPUT_DEFAULT_VALUE_FIELDS && MESSAGES.contains(message.getClass())) { - for (FieldDescriptor field : message.getDescriptorForType().getFields()) { - if (field.isOptional()) { - if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE - && !message.hasField(field)) { - // Always skip empty optional message fields. If not we will recurse indefinitely if - // a message has itself as a sub-field. - continue; - } - Descriptors.OneofDescriptor oneof = field.getContainingOneof(); - if (oneof != null && !message.hasField(field)) { - // Skip all oneof fields except the one that is actually set - continue; - } - } - if (!fieldsToPrint.containsKey(field)) { - fieldsToPrint.put(field, message.getField(field)); - } - } - } - - //for (Iterator> iter = message.getAllFields().entrySet() - for (Iterator> iter = fieldsToPrint.entrySet() - .iterator(); iter.hasNext(); ) { - Map.Entry field = iter.next(); - printField(field.getKey(), field.getValue(), generator, selfType); - if (iter.hasNext()) { - generator.print(","); - } - } - - // do not print unknown fields - // if (message.getUnknownFields().asMap().size() > 0) { - // generator.print(", "); - // } - // printUnknownFields(message.getUnknownFields(), generator, selfType); - } - - /** - * Like {@code print()}, but writes directly to a {@code String} and returns it. - */ - public static String printToString(Message message, boolean selfType) { - try { - StringBuilder text = new StringBuilder(); - print(message, text, selfType); - return text.toString(); - } catch (IOException e) { - throw new RuntimeException(WRITING_STRING_BUILDER_EXCEPTION, e); - } - } - - /** - * Like {@code print()}, but writes directly to a {@code String} and returns it. - */ - public static String printToString(Message message) { - try { - StringBuilder text = new StringBuilder(); - print(message, text, true); - return text.toString(); - } catch (IOException e) { - throw new RuntimeException(WRITING_STRING_BUILDER_EXCEPTION, e); - } - } - - /** - * Like {@code print()}, but writes directly to a {@code String} and returns it. - */ - public static String printToString(UnknownFieldSet fields, boolean selfType) { - try { - StringBuilder text = new StringBuilder(); - print(fields, text, selfType); - return text.toString(); - } catch (IOException e) { - throw new RuntimeException(WRITING_STRING_BUILDER_EXCEPTION, e); - } - } - - /** - * Parse a text-format message from {@code input} and merge the contents into {@code builder}. - */ - public static void merge(Readable input, Message.Builder builder) throws IOException { - merge(input, ExtensionRegistry.getEmptyRegistry(), builder, true); - } - - /** - * Parse a text-format message from {@code input} and merge the contents into {@code builder}. - */ - public static void merge(CharSequence input, Message.Builder builder) throws ParseException { - merge(input, ExtensionRegistry.getEmptyRegistry(), builder, true); - } - - /** - * Parse a text-format message from {@code input} and merge the contents into {@code builder}. - */ - public static void merge(Readable input, Message.Builder builder, boolean selfType) - throws IOException { - merge(input, ExtensionRegistry.getEmptyRegistry(), builder, selfType); - } - - /** - * Parse a text-format message from {@code input} and merge the contents into {@code builder}. - */ - public static void merge(CharSequence input, Message.Builder builder, boolean selfType) - throws ParseException { - merge(input, ExtensionRegistry.getEmptyRegistry(), builder, selfType); - } - - /** - * Parse a text-format message from {@code input} and merge the contents into {@code builder}. - * Extensions will be recognized if they are registered in {@code extensionRegistry}. - */ - public static void merge(Readable input, - ExtensionRegistry extensionRegistry, - Message.Builder builder, boolean selfType) throws IOException { - // Read the entire input to a String then parse that. - - // If StreamTokenizer were not quite so crippled, or if there were a kind - // of Reader that could read in chunks that match some particular regex, - // or if we wanted to write a custom Reader to tokenize our stream, then - // we would not have to read to one big String. Alas, none of these is - // the case. Oh well. - - merge(toStringBuilder(input), extensionRegistry, builder, selfType); - } - - /** - * Parse a text-format message from {@code input} and merge the contents into {@code builder}. - * Extensions will be recognized if they are registered in {@code extensionRegistry}. - */ - public static void merge(CharSequence input, - ExtensionRegistry extensionRegistry, - Message.Builder builder, boolean selfType) throws ParseException { - Tokenizer tokenizer = new Tokenizer(input); - - // Based on the state machine @ http://json.org/ - - tokenizer.consume("{"); // Needs to happen when the object starts. - while (!tokenizer.tryConsume("}")) { // Continue till the object is done - mergeField(tokenizer, extensionRegistry, builder, selfType); - } - // Test to make sure the tokenizer has reached the end of the stream. - if (!tokenizer.atEnd()) { - throw tokenizer.parseException( - "Expecting the end of the stream, but there seems to be more data! " - + "Check the input for a valid JSON format."); - } - } - - public static String printErrorMsg(Exception ex) { - StringBuilder text = new StringBuilder(); - text.append("{"); - text.append("\"Error\":"); - text.append("\""); - text.append(ex.getMessage()); - text.append("\""); - text.append("}"); - return text.toString(); - } - - public static void printField(FieldDescriptor field, Object value, JsonGenerator generator, - boolean selfType) - throws IOException { - - printSingleField(field, value, generator, selfType); - } - - private static void printSingleField(FieldDescriptor field, - Object value, JsonGenerator generator, boolean selfType) throws IOException { - if (field.isExtension()) { - generator.print("\""); - // We special-case MessageSet elements for compatibility with proto1. - if (field.getContainingType().getOptions().getMessageSetWireFormat() - && (field.getType() == FieldDescriptor.Type.MESSAGE) && (field.isOptional()) - // object equality - && (field.getExtensionScope() == field.getMessageType())) { - generator.print(field.getMessageType().getFullName()); - } else { - generator.print(field.getFullName()); - } - generator.print("\""); - } else { - generator.print("\""); - if (field.getType() == FieldDescriptor.Type.GROUP) { - // Groups must be serialized with their original capitalization. - generator.print(field.getMessageType().getName()); - } else { - generator.print(field.getName()); - } - generator.print("\""); - } - - // Done with the name, on to the value - - if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { - generator.print(": "); - generator.indent(); - } else { - generator.print(": "); - } - - if (field.isRepeated()) { - // Repeated field. Print each element. - generator.print("["); - for (Iterator iter = ((List) value).iterator(); iter.hasNext(); ) { - printFieldValue(field, iter.next(), generator, selfType); - if (iter.hasNext()) { - generator.print(","); - } - } - generator.print("]"); - } else { - printFieldValue(field, value, generator, selfType); - if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { - generator.outdent(); - } - } - } - - private static void printFieldValue(FieldDescriptor field, Object value, - JsonGenerator generator, boolean selfType) - throws IOException { - switch (field.getType()) { - case INT32: - case INT64: - case SINT32: - case SINT64: - case SFIXED32: - case SFIXED64: - case FLOAT: - case DOUBLE: - case BOOL: - // Good old toString() does what we want for these types. - generator.print(value.toString()); - break; - - case UINT32: - case FIXED32: - generator.print(unsignedToString((Integer) value)); - break; - - case UINT64: - case FIXED64: - generator.print(unsignedToString((Long) value)); - break; - - case STRING: - generator.print("\""); - generator.print(escapeText((String) value)); - generator.print("\""); - break; - - case BYTES: { - generator.print("\""); - generator.print(escapeBytes((ByteString) value, field.getFullName(), selfType)); - generator.print("\""); - break; - } - - case ENUM: { - generator.print("\""); - generator.print(((EnumValueDescriptor) value).getName()); - generator.print("\""); - break; - } - - case MESSAGE: - case GROUP: - generator.print("{"); - print((Message) value, generator, selfType); - generator.print("}"); - break; - default: - } - } - - protected static void printUnknownFields(UnknownFieldSet unknownFields, JsonGenerator generator, - boolean selfType) throws IOException { - boolean firstField = true; - for (Map.Entry entry : unknownFields.asMap().entrySet()) { - final UnknownFieldSet.Field field = entry.getValue(); - if (firstField) { - firstField = false; - } else { - generator.print(", "); - } - - generator.print("\""); - generator.print(entry.getKey().toString()); - generator.print("\""); - generator.print(": ["); - - boolean firstValue = true; - for (long value : field.getVarintList()) { - if (firstValue) { - firstValue = false; - } else { - generator.print(", "); - } - generator.print(unsignedToString(value)); - } - for (int value : field.getFixed32List()) { - if (firstValue) { - firstValue = false; - } else { - generator.print(", "); - } - generator.print("\""); - generator.print(String.format((Locale) null, "0x%08x", value)); - generator.print("\""); - } - for (long value : field.getFixed64List()) { - if (firstValue) { - firstValue = false; - } else { - generator.print(", "); - } - generator.print("\""); - generator.print(String.format((Locale) null, "0x%016x", value)); - generator.print("\""); - } - for (ByteString value : field.getLengthDelimitedList()) { - if (firstValue) { - firstValue = false; - } else { - generator.print(", "); - } - generator.print("\""); - generator.print(escapeBytes(value, "Hex", selfType)); //Just to HEX - generator.print("\""); - } - for (UnknownFieldSet value : field.getGroupList()) { - if (firstValue) { - firstValue = false; - } else { - generator.print(", "); - } - generator.print("{"); - printUnknownFields(value, generator, selfType); - generator.print("}"); - } - generator.print("]"); - } - } - - /** - * Convert an unsigned 32-bit integer to a string. - */ - private static String unsignedToString(int value) { - if (value >= 0) { - return Integer.toString(value); - } else { - return Long.toString((value) & 0x00000000FFFFFFFFL); - } - } - - // ================================================================= - // Parsing - - /** - * Convert an unsigned 64-bit integer to a string. - */ - private static String unsignedToString(long value) { - if (value >= 0) { - return Long.toString(value); - } else { - // Pull off the most-significant bit so that BigInteger doesn't think - // the number is negative, then set it again using setBit(). - return BigInteger.valueOf(value & 0x7FFFFFFFFFFFFFFFL).setBit(63).toString(); - } - } - - - // TODO(chrisn): See if working around java.io.Reader#read(CharBuffer) - // overhead is worthwhile - protected static StringBuilder toStringBuilder(Readable input) throws IOException { - StringBuilder text = new StringBuilder(); - CharBuffer buffer = CharBuffer.allocate(BUFFER_SIZE); - while (true) { - int n = input.read(buffer); - if (n == -1) { - break; - } - buffer.flip(); - text.append(buffer, 0, n); - } - return text; - } - - /** - * Parse a single field from {@code tokenizer} and merge it into {@code builder}. If a ',' is - * detected after the field ends, the next field will be parsed automatically - */ - protected static void mergeField(Tokenizer tokenizer, - ExtensionRegistry extensionRegistry, Message.Builder builder, - boolean selfType) throws ParseException { - FieldDescriptor field; - Descriptor type = builder.getDescriptorForType(); - final ExtensionRegistry.ExtensionInfo extension; - boolean unknown = false; - - String name = tokenizer.consumeIdentifier(); - field = type.findFieldByName(name); - - // Group names are expected to be capitalized as they appear in the - // .proto file, which actually matches their type names, not their field - // names. - if (field == null) { - // Explicitly specify US locale so that this code does not break when - // executing in Turkey. - String lowerName = name.toLowerCase(Locale.US); - field = type.findFieldByName(lowerName); - // If the case-insensitive match worked but the field is NOT a group, - if ((field != null) && (field.getType() != FieldDescriptor.Type.GROUP)) { - field = null; - } - } - // Again, special-case group names as described above. - if ((field != null) && (field.getType() == FieldDescriptor.Type.GROUP) - && !field.getMessageType().getName().equals(name)) { - field = null; - } - - // Last try to lookup by field-index if 'name' is numeric, - // which indicates a possible unknown field - if (field == null && DIGITS.matcher(name).matches()) { - field = type.findFieldByNumber(Integer.parseInt(name)); - unknown = true; - } - - // Finally, look for extensions - extension = extensionRegistry.findExtensionByName(name); - if (extension != null) { - if (extension.descriptor.getContainingType() != type) { - throw tokenizer.parseExceptionPreviousToken("Extension \"" + name - + "\" does not extend message type \"" - + type.getFullName() + "\"."); - } - field = extension.descriptor; - } - - // Disabled throwing exception if field not found, since it could be a different version. - if (field == null) { - handleMissingField(tokenizer, extensionRegistry, builder); - //throw tokenizer.parseExceptionPreviousToken("Message type \"" + type.getFullName() - // + "\" has no field named \"" + name - // + "\"."); - } - - if (field != null) { - tokenizer.consume(":"); - boolean array = tokenizer.tryConsume("["); - - if (array) { - while (!tokenizer.tryConsume("]")) { - handleValue(tokenizer, extensionRegistry, builder, field, extension, unknown, selfType); - tokenizer.tryConsume(","); - } - } else { - handleValue(tokenizer, extensionRegistry, builder, field, extension, unknown, selfType); - } - } - - if (tokenizer.tryConsume(",")) { - // Continue with the next field - mergeField(tokenizer, extensionRegistry, builder, selfType); - } - } - - private static void handleMissingField(Tokenizer tokenizer, - ExtensionRegistry extensionRegistry, - Message.Builder builder) throws ParseException { - tokenizer.tryConsume(":"); - if ("{".equals(tokenizer.currentToken())) { - // Message structure - tokenizer.consume("{"); - do { - tokenizer.consumeIdentifier(); - handleMissingField(tokenizer, extensionRegistry, builder); - } while (tokenizer.tryConsume(",")); - tokenizer.consume("}"); - } else if ("[".equals(tokenizer.currentToken())) { - // Collection - tokenizer.consume("["); - do { - handleMissingField(tokenizer, extensionRegistry, builder); - } while (tokenizer.tryConsume(",")); - tokenizer.consume("]"); - } else { //if (!",".equals(tokenizer.currentToken)){ - // Primitive value - if ("null".equals(tokenizer.currentToken())) { - tokenizer.consume("null"); - } else if (tokenizer.lookingAtInteger()) { - tokenizer.consumeInt64(); - } else if (tokenizer.lookingAtBoolean()) { - tokenizer.consumeBoolean(); - } else { - tokenizer.consumeString(); - } - } - } - - private static void handleValue(Tokenizer tokenizer, - ExtensionRegistry extensionRegistry, - Message.Builder builder, - FieldDescriptor field, - ExtensionRegistry.ExtensionInfo extension, - boolean unknown, boolean selfType) throws ParseException { - - Object value = null; - if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { - value = handleObject(tokenizer, extensionRegistry, builder, field, extension, unknown, - selfType); - } else { - value = handlePrimitive(tokenizer, field, selfType); - } - if (value != null) { - if (field.isRepeated()) { - builder.addRepeatedField(field, value); - } else { - builder.setField(field, value); - } - } - } - - private static Object handlePrimitive(Tokenizer tokenizer, FieldDescriptor field, - boolean selfType) - throws ParseException { - Object value = null; - if ("null".equals(tokenizer.currentToken())) { - tokenizer.consume("null"); - return value; - } - switch (field.getType()) { - case INT32: - case SINT32: - case SFIXED32: - value = tokenizer.consumeInt32(); - break; - - case INT64: - case SINT64: - case SFIXED64: - value = tokenizer.consumeInt64(); - break; - - case UINT32: - case FIXED32: - value = tokenizer.consumeUInt32(); - break; - - case UINT64: - case FIXED64: - value = tokenizer.consumeUInt64(); - break; - - case FLOAT: - value = tokenizer.consumeFloat(); - break; - - case DOUBLE: - value = tokenizer.consumeDouble(); - break; - - case BOOL: - value = tokenizer.consumeBoolean(); - break; - - case STRING: - value = tokenizer.consumeString(); - break; - - case BYTES: - value = tokenizer.consumeByteString(field.getFullName(), selfType); - break; - - case ENUM: { - EnumDescriptor enumType = field.getEnumType(); - - if (tokenizer.lookingAtInteger()) { - int number = tokenizer.consumeInt32(); - value = enumType.findValueByNumber(number); - if (value == null) { - throw tokenizer.parseExceptionPreviousToken("Enum type \"" - + enumType.getFullName() - + "\" has no value with number " - + number + "."); - } - } else { - String id = tokenizer.consumeIdentifier(); - if (StringUtils.isAllLowerCase(id)) { - char b = id.charAt(0); - b = (char) (b + 'A' - 'a'); - String s = id.substring(1); - id = b + s; - } - value = enumType.findValueByName(id); - if (value == null) { - throw tokenizer.parseExceptionPreviousToken("Enum type \"" - + enumType.getFullName() - + "\" has no value named \"" - + id + "\"."); - } - } - - break; - } - - case MESSAGE: - case GROUP: - throw new RuntimeException("Can't get here."); - default: - } - return value; - } - - private static Object handleObject(Tokenizer tokenizer, - ExtensionRegistry extensionRegistry, - Message.Builder builder, - FieldDescriptor field, - ExtensionRegistry.ExtensionInfo extension, - boolean unknown, boolean selfType) throws ParseException { - - Message.Builder subBuilder; - if (extension == null) { - subBuilder = builder.newBuilderForField(field); - } else { - subBuilder = extension.defaultInstance.newBuilderForType(); - } - - if (unknown) { - ByteString data = tokenizer.consumeByteString("", selfType); - try { - subBuilder.mergeFrom(data); - return subBuilder.build(); - } catch (InvalidProtocolBufferException e) { - throw tokenizer.parseException("Failed to build " + field.getFullName() + " from " + data); - } - } - - tokenizer.consume("{"); - String endToken = "}"; - - while (!tokenizer.tryConsume(endToken)) { - if (tokenizer.atEnd()) { - throw tokenizer.parseException("Expected \"" + endToken + "\"."); - } - mergeField(tokenizer, extensionRegistry, subBuilder, selfType); - if (tokenizer.tryConsume(",")) { - // there are more fields in the object, so continue - continue; - } - } - - return subBuilder.build(); - } - - /** - * Escapes bytes in the format used in protocol buffer text format, which is the same as the - * format used for C string literals. All bytes that are not printable 7-bit ASCII characters are - * escaped, as well as backslash, single-quote, and double-quote characters. Characters for which - * no defined short-hand escape sequence is defined will be escaped using 3-digit octal - * sequences. - */ - static String escapeBytes(ByteString input) { - return ByteArray.toHexString(input.toByteArray()); - } - - static String escapeBytes(ByteString input, final String fliedName, boolean selfType) { - if (!selfType) { - return ByteArray.toHexString(input.toByteArray()); - } else { - return escapeBytesSelfType(input, fliedName); - } - } - - static String escapeBytesSelfType(ByteString input, final String fliedName) { - //Address - if (HttpSelfFormatFieldName.isAddressFormat(fliedName)) { - return StringUtil.encode58Check(input.toByteArray()); - } - //Normal String - if (HttpSelfFormatFieldName.isNameStringFormat(fliedName)) { - String result = new String(input.toByteArray()); - result = result.replaceAll("\"", "\\\\\""); - try { - JSON.parseObject("{\"key\":\"" + result + "\"}"); - return result; - } catch (Exception e) { - return ByteArray.toHexString(input.toByteArray()); - } - } - //HEX - return ByteArray.toHexString(input.toByteArray()); - } - - static String unicodeEscaped(char ch) { - if (ch < 0x10) { - return "\\u000" + Integer.toHexString(ch); - } else if (ch < 0x100) { - return "\\u00" + Integer.toHexString(ch); - } else if (ch < 0x1000) { - return "\\u0" + Integer.toHexString(ch); - } - return "\\u" + Integer.toHexString(ch); - } - - static ByteString unescapeBytes(CharSequence input) throws InvalidEscapeSequence { - try { - return ByteString.copyFrom(ByteArray.fromHexString(input.toString())); - } catch (Exception e) { - throw new InvalidEscapeSequence("invalidate hex String"); - } - } - - // ================================================================= - // Utility functions - // - // Some of these methods are package-private because Descriptors.java uses - // them. - - /** - * Implements JSON string escaping as specified here. - *

  • The following characters are escaped by prefixing them with a '\' : - * \b,\f,\n,\r,\t,\,"
  • Other control characters in the range 0x0000-0x001F are escaped - * using the \\uXXXX notation
  • UTF-16 surrogate pairs are encoded using the \\uXXXX\\uXXXX - * notation
  • any other character is printed as-is
- */ - static String escapeText(String input) { - StringBuilder builder = new StringBuilder(input.length()); - CharacterIterator iter = new StringCharacterIterator(input); - for (char c = iter.first(); c != CharacterIterator.DONE; c = iter.next()) { - switch (c) { - case '\b': - builder.append("\\b"); - break; - case '\f': - builder.append("\\f"); - break; - case '\n': - builder.append("\\n"); - break; - case '\r': - builder.append("\\r"); - break; - case '\t': - builder.append("\\t"); - break; - case '\\': - builder.append("\\\\"); - break; - case '"': - builder.append("\\\""); - break; - default: - // Check for other control characters - if (c >= 0x0000 && c <= 0x001F) { - appendEscapedUnicode(builder, c); - } else if (Character.isHighSurrogate(c)) { - // Encode the surrogate pair using 2 six-character sequence (\\uXXXX\\uXXXX) - appendEscapedUnicode(builder, c); - c = iter.next(); - if (c == CharacterIterator.DONE) { - throw new IllegalArgumentException( - "invalid unicode string: unexpected high surrogate pair value " - + "without corresponding low value."); - } - appendEscapedUnicode(builder, c); - } else { - // Anything else can be printed as-is - builder.append(c); - } - break; - } - } - return builder.toString(); - } - - static void appendEscapedUnicode(StringBuilder builder, char ch) { - String prefix = "\\u"; - if (ch < 0x10) { - prefix = "\\u000"; - } else if (ch < 0x100) { - prefix = "\\u00"; - } else if (ch < 0x1000) { - prefix = "\\u0"; - } - builder.append(prefix).append(Integer.toHexString(ch)); - } - - /** - * Un-escape a text string as escaped using {@link #escapeText(String)}. - */ - static String unescapeText(String input) throws InvalidEscapeSequence { - StringBuilder builder = new StringBuilder(); - char[] array = input.toCharArray(); - for (int i = 0; i < array.length; i++) { - char c = array[i]; - if (c == '\\') { - if (i + 1 < array.length) { - ++i; - c = array[i]; - switch (c) { - case 'b': - builder.append('\b'); - break; - case 'f': - builder.append('\f'); - break; - case 'n': - builder.append('\n'); - break; - case 'r': - builder.append('\r'); - break; - case 't': - builder.append('\t'); - break; - case '\\': - builder.append('\\'); - break; - case '"': - builder.append('\"'); - break; - case '\'': - builder.append('\''); - break; - case 'u': - // read the next 4 chars - if (i + 4 < array.length) { - ++i; - int code = Integer.parseInt(new String(array, i, 4), 16); - // this cast is safe because we know how many chars we read - builder.append((char) code); - i += 3; - } else { - throw new InvalidEscapeSequence("Invalid escape sequence: '\\u' at end of string."); - } - break; - default: - throw new InvalidEscapeSequence("Invalid escape sequence: '\\" + c + "'"); - } - } else { - throw new InvalidEscapeSequence("Invalid escape sequence: '\\' at end of string."); - } - } else { - builder.append(c); - } - } - - return builder.toString(); - } - - /** - * Is this an octal digit. - */ - private static boolean isOctal(char c) { - return ('0' <= c) && (c <= '7'); - } - - /** - * Is this a hex digit. - */ - private static boolean isHex(char c) { - return (('0' <= c) && (c <= '9')) || (('a' <= c) && (c <= 'f')) - || (('A' <= c) && (c <= 'F')); - } - - /** - * Interpret a character as a digit (in any base up to 36) and return the numeric value. This is - * like {@code Character.digit()} but we don't accept non-ASCII digits. - */ - private static int digitValue(char c) { - if (('0' <= c) && (c <= '9')) { - return c - '0'; - } else if (('a' <= c) && (c <= 'z')) { - return c - 'a' + 10; - } else { - return c - 'A' + 10; - } - } - - /** - * Parse a 32-bit signed integer from the text. Unlike the Java standard {@code - * Integer.parseInt()}, this function recognizes the prefixes "0x" and "0" to signify hexidecimal - * and octal numbers, respectively. - */ - static int parseInt32(String text) throws NumberFormatException { - return (int) parseInteger(text, true, false); - } - - /** - * Parse a 32-bit unsigned integer from the text. Unlike the Java standard {@code - * Integer.parseInt()}, this function recognizes the prefixes "0x" and "0" to signify hexidecimal - * and octal numbers, respectively. The result is coerced to a (signed) {@code int} when returned - * since Java has no unsigned integer type. - */ - static int parseUInt32(String text) throws NumberFormatException { - return (int) parseInteger(text, false, false); - } - - /** - * Parse a 64-bit signed integer from the text. Unlike the Java standard {@code - * Integer.parseInt()}, this function recognizes the prefixes "0x" and "0" to signify hexidecimal - * and octal numbers, respectively. - */ - static long parseInt64(String text) throws NumberFormatException { - return parseInteger(text, true, true); - } - - /** - * Parse a 64-bit unsigned integer from the text. Unlike the Java standard {@code - * Integer.parseInt()}, this function recognizes the prefixes "0x" and "0" to signify hexidecimal - * and octal numbers, respectively. The result is coerced to a (signed) {@code long} when returned - * since Java has no unsigned long type. - */ - static long parseUInt64(String text) throws NumberFormatException { - return parseInteger(text, false, true); - } - - private static long parseInteger(String text, boolean isSigned, boolean isLong) - throws NumberFormatException { - int pos = 0; - - boolean negative = false; - if (text.startsWith("-", pos)) { - if (!isSigned) { - throw new NumberFormatException("Number must be positive: " + text); - } - ++pos; - negative = true; - } - - int radix = 10; - if (text.startsWith("0x", pos)) { - pos += 2; - radix = 16; - } else if (text.startsWith("0", pos)) { - radix = 8; - } - - String numberText = text.substring(pos); - - long result = 0; - if (numberText.length() < 16) { - // Can safely assume no overflow. - result = Long.parseLong(numberText, radix); - if (negative) { - result = -result; - } - - // Check bounds. - // No need to check for 64-bit numbers since they'd have to be 16 chars - // or longer to overflow. - if (!isLong) { - if (isSigned) { - if ((result > Integer.MAX_VALUE) || (result < Integer.MIN_VALUE)) { - throw new NumberFormatException("Number out of range for 32-bit signed integer: " - + text); - } - } else { - if ((result >= (1L << 32)) || (result < 0)) { - throw new NumberFormatException("Number out of range for 32-bit unsigned integer: " - + text); - } - } - } - } else { - BigInteger bigValue = new BigInteger(numberText, radix); - if (negative) { - bigValue = bigValue.negate(); - } - - // Check bounds. - if (!isLong) { - if (isSigned) { - if (bigValue.bitLength() > 31) { - throw new NumberFormatException("Number out of range for 32-bit signed integer: " - + text); - } - } else { - if (bigValue.bitLength() > 32) { - throw new NumberFormatException("Number out of range for 32-bit unsigned integer: " - + text); - } - } - } else { - if (isSigned) { - if (bigValue.bitLength() > 63) { - throw new NumberFormatException("Number out of range for 64-bit signed integer: " - + text); - } - } else { - if (bigValue.bitLength() > 64) { - throw new NumberFormatException("Number out of range for 64-bit unsigned integer: " - + text); - } - } - } - - result = bigValue.longValueExact(); - } - - return result; - } - - /** - * An inner class for writing text to the output stream. - */ - protected static class JsonGenerator { - - private Appendable output; - private boolean atStartOfLine = true; - private StringBuilder indent = new StringBuilder(); - - public JsonGenerator(Appendable output) { - this.output = output; - } - - /** - * Indent text by two spaces. After calling Indent(), two spaces will be inserted at the - * beginning of each line of text. Indent() may be called multiple times to produce deeper - * indents. - */ - public void indent() { - indent.append(" "); - } - - /** - * Reduces the current indent level by two spaces, or crashes if the indent level is zero. - */ - public void outdent() { - int length = indent.length(); - if (length == 0) { - throw new IllegalArgumentException(" Outdent() without matching Indent()."); - } - indent.delete(length - 2, length); - } - - /** - * Print text to the output stream. - */ - public void print(CharSequence text) throws IOException { - int size = text.length(); - int pos = 0; - - for (int i = 0; i < size; i++) { - if (text.charAt(i) == '\n') { - write(text.subSequence(pos, size), i - pos + 1); - pos = i + 1; - atStartOfLine = true; - } - } - write(text.subSequence(pos, size), size - pos); - } - - private void write(CharSequence data, int size) throws IOException { - if (size == 0) { - return; - } - if (atStartOfLine) { - atStartOfLine = false; - output.append(indent); - } - output.append(data); - } - } - - /** - * Represents a stream of tokens parsed from a {@code String}. - *

- * - *

The Java standard library provides many classes that you might think would be useful for - * implementing this, but aren't. For example: - *

- *

  • {@code java.io.StreamTokenizer}: This almost does what we want -- or, at least, - * something that would get us close to what we want -- except for one fatal flaw: It - * automatically un-escapes strings using Java escape sequences, which do not include all the - * escape sequences we need to support (e.g. '\x').
  • {@code java.util.Scanner}: This seems like - * a great way at least to parse regular expressions out of a stream (so we wouldn't have to load - * the entire input into a single string before parsing). Sadly, {@code Scanner} requires that - * tokens be delimited with some delimiter. Thus, although the text "foo:" should parse to two - * tokens ("foo" and ":"), {@code Scanner} would recognize it only as a single token. Furthermore, - * {@code Scanner} provides no way to inspect the contents of delimiters, making it impossible to - * keep track of line and column numbers.
- *

- * - *

Luckily, Java's regular expression support does manage to be useful to us. (Barely: We need - * {@code Matcher.usePattern()}, which is new in Java 1.5.) So, we can use that, at least. - * Unfortunately, this implies that we need to have the entire input in one contiguous string. - */ - protected static class Tokenizer { - - // We use possesive quantifiers (*+ and ++) because otherwise the Java - // regex matcher has stack overflows on large inputs. - private static final Pattern WHITESPACE = - Pattern.compile("(\\s|(#.*$))++", Pattern.MULTILINE); - private static final Pattern TOKEN = Pattern.compile( - "[a-zA-Z_][0-9a-zA-Z_+-]*+|" + // an identifier - "[.]?[0-9+-][0-9a-zA-Z_.+-]*+|" + // a number - "\"([^\"\n\\\\]|\\\\.)*+(\"|\\\\?$)|" + // a double-quoted string - "\'([^\'\n\\\\]|\\\\.)*+(\'|\\\\?$)", // a single-quoted string - Pattern.MULTILINE); - private static final Pattern DOUBLE_INFINITY = Pattern.compile( - "-?inf(inity)?", - Pattern.CASE_INSENSITIVE); - private static final Pattern FLOAT_INFINITY = Pattern.compile( - "-?inf(inity)?f?", - Pattern.CASE_INSENSITIVE); - private static final Pattern FLOAT_NAN = Pattern.compile( - "nanf?", - Pattern.CASE_INSENSITIVE); - private final CharSequence text; - private final Matcher matcher; - private String currentToken; - // The character index within this.text at which the current token begins. - private int pos = 0; - // The line and column numbers of the current token. - private int line = 0; - private int column = 0; - // The line and column numbers of the previous token (allows throwing - // errors *after* consuming). - private int previousLine = 0; - private int previousColumn = 0; - - /** - * Construct a tokenizer that parses tokens from the given text. - */ - public Tokenizer(CharSequence text) { - this.text = text; - matcher = WHITESPACE.matcher(text); - skipWhitespace(); - nextToken(); - } - - static ByteString unescapeBytes(CharSequence input) throws InvalidEscapeSequence { - try { - return ByteString.copyFrom(ByteArray.fromHexString(input.toString())); - } catch (Exception e) { - throw new InvalidEscapeSequence("INVALID hex String"); - } - } - - static ByteString unescapeBytesSelfType(String input, final String fliedName) - throws InvalidEscapeSequence { - //Address base58 -> ByteString - if (HttpSelfFormatFieldName.isAddressFormat(fliedName)) { - return ByteString.copyFrom(Commons.decodeFromBase58Check(input)); - } - - //Normal String -> ByteString - if (HttpSelfFormatFieldName.isNameStringFormat(fliedName)) { - return ByteString.copyFromUtf8(input); - } - - return unescapeBytes(input); - } - - /** - * Are we at the end of the input. - */ - public boolean atEnd() { - return currentToken.length() == 0; - } - - /** - * Advance to the next token. - */ - public void nextToken() { - previousLine = line; - previousColumn = column; - - // Advance the line counter to the current position. - while (pos < matcher.regionStart()) { - if (text.charAt(pos) == '\n') { - ++line; - column = 0; - } else { - ++column; - } - ++pos; - } - - // Match the next token. - if (matcher.regionStart() == matcher.regionEnd()) { - // EOF - currentToken = ""; - } else { - matcher.usePattern(TOKEN); - if (matcher.lookingAt()) { - currentToken = matcher.group(); - matcher.region(matcher.end(), matcher.regionEnd()); - } else { - // Take one character. - currentToken = String.valueOf(text.charAt(pos)); - matcher.region(pos + 1, matcher.regionEnd()); - } - - skipWhitespace(); - } - } - - /** - * Skip over any whitespace so that the matcher region starts at the next token. - */ - private void skipWhitespace() { - matcher.usePattern(WHITESPACE); - if (matcher.lookingAt()) { - matcher.region(matcher.end(), matcher.regionEnd()); - } - } - - /** - * If the next token exactly matches {@code token}, consume it and return {@code true}. - * Otherwise, return {@code false} without doing anything. - */ - public boolean tryConsume(String token) { - if (currentToken.equals(token)) { - nextToken(); - return true; - } else { - return false; - } - } - - /** - * If the next token exactly matches {@code token}, consume it. Otherwise, throw a {@link - * ParseException}. - */ - public void consume(String token) throws ParseException { - if (!tryConsume(token)) { - throw parseException("Expected \"" + token + "\"."); - } - } - - /** - * Returns {@code true} if the next token is an integer, but does not consume it. - */ - public boolean lookingAtInteger() { - if (currentToken.length() == 0) { - return false; - } - - char c = currentToken.charAt(0); - return (('0' <= c) && (c <= '9')) || (c == '-') || (c == '+'); - } - - /** - * Returns {@code true} if the next token is a boolean (true/false), but does not consume it. - */ - public boolean lookingAtBoolean() { - if (currentToken.length() == 0) { - return false; - } - - return ("true".equals(currentToken) || "false".equals(currentToken)); - } - - /** - * @return currentToken to which the Tokenizer is pointing. - */ - public String currentToken() { - return currentToken; - } - - /** - * If the next token is an identifier, consume it and return its value. Otherwise, throw a - * {@link ParseException}. - */ - public String consumeIdentifier() throws ParseException { - for (int i = 0; i < currentToken.length(); i++) { - char c = currentToken.charAt(i); - if ((('a' <= c) && (c <= 'z')) || (('A' <= c) && (c <= 'Z')) - || (('0' <= c) && (c <= '9')) || (c == '_') || (c == '.') || (c == '"')) { - // OK - } else { - throw parseException("Expected identifier. -" + c); - } - } - - String result = currentToken; - // Need to clean-up result to remove quotes of any kind - result = result.replaceAll("\"|'", ""); - nextToken(); - return result; - } - - /** - * If the next token is a 32-bit signed integer, consume it and return its value. Otherwise, - * throw a {@link ParseException}. - */ - public int consumeInt32() throws ParseException { - try { - int result = parseInt32(currentToken); - nextToken(); - return result; - } catch (NumberFormatException e) { - throw integerParseException(e); - } - } - - /** - * If the next token is a 32-bit unsigned integer, consume it and return its value. Otherwise, - * throw a {@link ParseException}. - */ - public int consumeUInt32() throws ParseException { - try { - int result = parseUInt32(currentToken); - nextToken(); - return result; - } catch (NumberFormatException e) { - throw integerParseException(e); - } - } - - /** - * If the next token is a 64-bit signed integer, consume it and return its value. Otherwise, - * throw a {@link ParseException}. - */ - public long consumeInt64() throws ParseException { - try { - long result = parseInt64(currentToken); - nextToken(); - return result; - } catch (NumberFormatException e) { - throw integerParseException(e); - } - } - - /** - * If the next token is a 64-bit unsigned integer, consume it and return its value. Otherwise, - * throw a {@link ParseException}. - */ - public long consumeUInt64() throws ParseException { - try { - long result = parseUInt64(currentToken); - nextToken(); - return result; - } catch (NumberFormatException e) { - throw integerParseException(e); - } - } - - /** - * If the next token is a double, consume it and return its value. Otherwise, throw a {@link - * ParseException}. - */ - public double consumeDouble() throws ParseException { - // We need to parse infinity and nan separately because - // Double.parseDouble() does not accept "inf", "infinity", or "nan". - if (DOUBLE_INFINITY.matcher(currentToken).matches()) { - boolean negative = currentToken.startsWith("-"); - nextToken(); - return negative ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY; - } - if (currentToken.equalsIgnoreCase("nan")) { - nextToken(); - return Double.NaN; - } - try { - double result = Double.parseDouble(currentToken); - nextToken(); - return result; - } catch (NumberFormatException e) { - throw floatParseException(e); - } - } - - /** - * If the next token is a float, consume it and return its value. Otherwise, throw a {@link - * ParseException}. - */ - public float consumeFloat() throws ParseException { - // We need to parse infinity and nan separately because - // Float.parseFloat() does not accept "inf", "infinity", or "nan". - if (FLOAT_INFINITY.matcher(currentToken).matches()) { - boolean negative = currentToken.startsWith("-"); - nextToken(); - return negative ? Float.NEGATIVE_INFINITY : Float.POSITIVE_INFINITY; - } - if (FLOAT_NAN.matcher(currentToken).matches()) { - nextToken(); - return Float.NaN; - } - try { - float result = Float.parseFloat(currentToken); - nextToken(); - return result; - } catch (NumberFormatException e) { - throw floatParseException(e); - } - } - - /** - * If the next token is a boolean, consume it and return its value. Otherwise, throw a {@link - * ParseException}. - */ - public boolean consumeBoolean() throws ParseException { - if (currentToken.equals("true")) { - nextToken(); - return true; - } else if (currentToken.equals("false")) { - nextToken(); - return false; - } else { - throw parseException("Expected \"true\" or \"false\"."); - } - } - - /** - * If the next token is a string, consume it and return its (unescaped) value. Otherwise, throw - * a {@link ParseException}. - */ - public String consumeString() throws ParseException { - char quote = currentToken.length() > 0 ? currentToken.charAt(0) : '\0'; - if ((quote != '\"') && (quote != '\'')) { - throw parseException(EXPECTED_STRING); - } - - if ((currentToken.length() < 2) - || (currentToken.charAt(currentToken.length() - 1) != quote)) { - throw parseException(MISSING_END_QUOTE); - } - - try { - String escaped = currentToken.substring(1, currentToken.length() - 1); - String result = unescapeText(escaped); - nextToken(); - return result; - } catch (InvalidEscapeSequence e) { - throw parseException(e.getMessage()); - } - } - - public ByteString consumeByteString() throws ParseException { - char quote = currentToken.length() > 0 ? currentToken.charAt(0) : '\0'; - if ((quote != '\"') && (quote != '\'')) { - throw parseException(EXPECTED_STRING); - } - - if ((currentToken.length() < 2) - || (currentToken.charAt(currentToken.length() - 1) != quote)) { - throw parseException(MISSING_END_QUOTE); - } - - try { - String escaped = currentToken.substring(1, currentToken.length() - 1); - ByteString result = unescapeBytes(escaped); - nextToken(); - return result; - } catch (InvalidEscapeSequence e) { - throw parseException(e.getMessage()); - } - } - - public ByteString consumeByteString(final String fieldName, boolean selfType) - throws ParseException { - char quote = currentToken.length() > 0 ? currentToken.charAt(0) : '\0'; - if ((quote != '\"') && (quote != '\'')) { - throw parseException(EXPECTED_STRING); - } - - if ((currentToken.length() < 2) - || (currentToken.charAt(currentToken.length() - 1) != quote)) { - throw parseException(MISSING_END_QUOTE); - } - - try { - String escaped = currentToken.substring(1, currentToken.length() - 1); - ByteString result; - if (!selfType) { - result = unescapeBytes(escaped); - } else { - result = unescapeBytesSelfType(escaped, fieldName); - } - nextToken(); - return result; - } catch (InvalidEscapeSequence e) { - throw parseException(e.getMessage()); - } catch (IllegalArgumentException e) { - throw parseException("INVALID base58 String, " + e.getMessage()); - } - } - - /** - * Returns a {@link ParseException} with the current line and column numbers in the description, - * suitable for throwing. - */ - public ParseException parseException(String description) { - // Note: People generally prefer one-based line and column numbers. - return new ParseException((line + 1) + ":" + (column + 1) + ": " + description); - } - - /** - * Returns a {@link ParseException} with the line and column numbers of the previous token in - * the description, suitable for throwing. - */ - public ParseException parseExceptionPreviousToken(String description) { - // Note: People generally prefer one-based line and column numbers. - return new ParseException((previousLine + 1) + ":" + (previousColumn + 1) + ": " - + description); - } - - /** - * Constructs an appropriate {@link ParseException} for the given {@code NumberFormatException} - * when trying to parse an integer. - */ - private ParseException integerParseException(NumberFormatException e) { - return parseException("Couldn't parse integer: " + e.getMessage()); - } - - /** - * Constructs an appropriate {@link ParseException} for the given {@code NumberFormatException} - * when trying to parse a float or double. - */ - private ParseException floatParseException(NumberFormatException e) { - return parseException("Couldn't parse number: " + e.getMessage()); - } - } - - /** - * Thrown when parsing an invalid text format message. - */ - public static class ParseException extends IOException { - - private static final long serialVersionUID = 1L; - - public ParseException(String message) { - super(message); - } - } - - /** - * Thrown by {@link JsonFormat#unescapeBytes} and {@link JsonFormat#unescapeText} when an invalid - * escape sequence is seen. - */ - static class InvalidEscapeSequence extends IOException { - - private static final long serialVersionUID = 1L; - - public InvalidEscapeSequence(String description) { - super(description); - } - } -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/MarketUtils.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/MarketUtils.java deleted file mode 100644 index a36fc8ad..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/MarketUtils.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.tron.plugins.utils; - -import java.math.BigInteger; -import org.tron.plugins.utils.ByteArray; - -public class MarketUtils { - - public static final int TOKEN_ID_LENGTH = ByteArray - .fromString(Long.toString(Long.MAX_VALUE)).length; // 19 - - - - /** - * In order to avoid the difference between the data of same key stored and fetched by hashMap and - * levelDB, when creating the price key, we will find the GCD (Greatest Common Divisor) of - * sellTokenQuantity and buyTokenQuantity. - */ - public static byte[] createPairPriceKey(byte[] sellTokenId, byte[] buyTokenId, - long sellTokenQuantity, long buyTokenQuantity) { - - byte[] sellTokenQuantityBytes; - byte[] buyTokenQuantityBytes; - - // cal the GCD - long gcd = findGCD(sellTokenQuantity, buyTokenQuantity); - if (gcd == 0) { - sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity); - buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity); - } else { - sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity / gcd); - buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity / gcd); - } - - return doCreatePairPriceKey(sellTokenId, buyTokenId, - sellTokenQuantityBytes, buyTokenQuantityBytes); - } - - public static long findGCD(long number1, long number2) { - if (number1 == 0 || number2 == 0) { - return 0; - } - return calGCD(number1, number2); - } - - private static long calGCD(long number1, long number2) { - if (number2 == 0) { - return number1; - } - return calGCD(number2, number1 % number2); - } - - - private static byte[] doCreatePairPriceKey(byte[] sellTokenId, byte[] buyTokenId, - byte[] sellTokenQuantity, byte[] buyTokenQuantity) { - byte[] result = new byte[TOKEN_ID_LENGTH + TOKEN_ID_LENGTH - + sellTokenQuantity.length + buyTokenQuantity.length]; - - System.arraycopy(sellTokenId, 0, result, 0, sellTokenId.length); - System.arraycopy(buyTokenId, 0, result, TOKEN_ID_LENGTH, buyTokenId.length); - System.arraycopy(sellTokenQuantity, 0, result, - TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, - sellTokenQuantity.length); - System.arraycopy(buyTokenQuantity, 0, result, - TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + buyTokenQuantity.length, - buyTokenQuantity.length); - - return result; - } - - - public static int comparePriceKey(byte[] o1, byte[] o2) { - //compare pair - byte[] pair1 = new byte[TOKEN_ID_LENGTH * 2]; - byte[] pair2 = new byte[TOKEN_ID_LENGTH * 2]; - - System.arraycopy(o1, 0, pair1, 0, TOKEN_ID_LENGTH * 2); - System.arraycopy(o2, 0, pair2, 0, TOKEN_ID_LENGTH * 2); - - int pairResult = ByteArray.compareUnsigned(pair1, pair2); - if (pairResult != 0) { - return pairResult; - } - - //compare price - byte[] getSellTokenQuantity1 = new byte[8]; - byte[] getBuyTokenQuantity1 = new byte[8]; - - byte[] getSellTokenQuantity2 = new byte[8]; - byte[] getBuyTokenQuantity2 = new byte[8]; - - int longByteNum = 8; - - System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, - getSellTokenQuantity1, 0, longByteNum); - System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, - getBuyTokenQuantity1, 0, longByteNum); - - System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, - getSellTokenQuantity2, 0, longByteNum); - System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, - getBuyTokenQuantity2, 0, longByteNum); - - long sellTokenQuantity1 = ByteArray.toLong(getSellTokenQuantity1); - long buyTokenQuantity1 = ByteArray.toLong(getBuyTokenQuantity1); - long sellTokenQuantity2 = ByteArray.toLong(getSellTokenQuantity2); - long buyTokenQuantity2 = ByteArray.toLong(getBuyTokenQuantity2); - - if ((sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) - && (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0)) { - return 0; - } - - if (sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) { - return -1; - } - - if (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0) { - return 1; - } - - return comparePrice(sellTokenQuantity1, buyTokenQuantity1, - sellTokenQuantity2, buyTokenQuantity2); - - } - - /** - * Note: the params should be the same token pair, or you should change the order. - * All the quantity should be bigger than 0. - * */ - public static int comparePrice(long price1SellQuantity, long price1BuyQuantity, - long price2SellQuantity, long price2BuyQuantity) { - try { - return Long.compare(Math.multiplyExact(price1BuyQuantity, price2SellQuantity), - Math.multiplyExact(price2BuyQuantity, price1SellQuantity)); - - } catch (ArithmeticException ex) { - // do nothing here, because we will use BigInteger to compute again - } - - BigInteger price1BuyQuantityBI = BigInteger.valueOf(price1BuyQuantity); - BigInteger price1SellQuantityBI = BigInteger.valueOf(price1SellQuantity); - BigInteger price2BuyQuantityBI = BigInteger.valueOf(price2BuyQuantity); - BigInteger price2SellQuantityBI = BigInteger.valueOf(price2SellQuantity); - - return price1BuyQuantityBI.multiply(price2SellQuantityBI) - .compareTo(price2BuyQuantityBI.multiply(price1SellQuantityBI)); - } -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBInterface.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBInterface.java deleted file mode 100644 index 513e021c..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBInterface.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.tron.plugins.utils.db; - -import java.io.Closeable; -import java.io.IOException; - - -public interface DBInterface extends Closeable { - - byte[] get(byte[] key); - - void put(byte[] key, byte[] value); - - void delete(byte[] key); - - DBIterator iterator(); - - long size(); - - void close() throws IOException; - - String getName(); - -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java deleted file mode 100644 index 50957bbe..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.tron.plugins.utils.db; - -import java.io.IOException; -import lombok.Getter; -import org.rocksdb.RocksDBException; -import org.rocksdb.RocksIterator; - -public class RocksDBImpl implements DBInterface { - - private org.rocksdb.RocksDB rocksDB; - - @Getter - private final String name; - - public RocksDBImpl(org.rocksdb.RocksDB rocksDB, String name) { - this.rocksDB = rocksDB; - this.name = name; - } - - @Override - public byte[] get(byte[] key) { - try { - return rocksDB.get(key); - } catch (RocksDBException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public void put(byte[] key, byte[] value) { - try { - rocksDB.put(key, value); - } catch (RocksDBException e) { - e.printStackTrace(); - } - } - - @Override - public void delete(byte[] key) { - try { - rocksDB.delete(key); - } catch (RocksDBException e) { - e.printStackTrace(); - } - } - - @Override - public DBIterator iterator() { - return new RockDBIterator(rocksDB.newIterator( - new org.rocksdb.ReadOptions().setFillCache(false))); - } - - @Override - public long size() { - RocksIterator iterator = rocksDB.newIterator(); - long size = 0; - for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { - size++; - } - iterator.close(); - return size; - } - - @Override - public void close() throws IOException { - rocksDB.close(); - } -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/ArchiveManifest.java b/tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java similarity index 98% rename from tools/toolkit/src/main/java/org/tron/plugins/ArchiveManifest.java rename to tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java index 7d638bcd..1d7a9102 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/ArchiveManifest.java +++ b/tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java @@ -35,6 +35,7 @@ import org.iq80.leveldb.DB; import org.iq80.leveldb.Options; import org.iq80.leveldb.impl.Filename; +import org.tron.plugins.utils.DBUtils; import picocli.CommandLine; import picocli.CommandLine.Option; @@ -183,7 +184,7 @@ public boolean checkManifest(String dir) throws IOException { return false; } logger.info("CurrentName {}/{},size {} kb.", dir, currentName, current.length() / 1024); - if ("market_pair_price_to_order".equalsIgnoreCase(this.name)) { + if (DBUtils.MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(this.name)) { logger.info("Db {} ignored.", this.name); return false; } @@ -268,4 +269,4 @@ public static class Args { @Option(names = {"-h", "--help"}, help = true) private boolean help; } -} +} \ No newline at end of file diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbArchive.java b/tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java similarity index 98% rename from tools/toolkit/src/main/java/org/tron/plugins/DbArchive.java rename to tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java index 2f117cd4..15bb281b 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbArchive.java +++ b/tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java @@ -19,6 +19,7 @@ import org.iq80.leveldb.DB; import org.iq80.leveldb.Options; import org.iq80.leveldb.impl.Filename; +import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.FileUtils; import picocli.CommandLine; import picocli.CommandLine.Option; @@ -163,7 +164,7 @@ public boolean checkManifest(String dir) throws IOException { return false; } logger.info("CurrentName {}/{},size {} kb.", dir, currentName, current.length() / 1024); - if ("market_pair_price_to_order".equalsIgnoreCase(this.name)) { + if (DBUtils.MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(this.name)) { logger.info("Db {} ignored.", this.name); return false; } @@ -195,4 +196,4 @@ public void doArchive() { } } -} +} \ No newline at end of file diff --git a/tools/toolkit/src/main/resources/fork.conf b/tools/toolkit/src/main/resources/fork.conf deleted file mode 100644 index f99896e3..00000000 --- a/tools/toolkit/src/main/resources/fork.conf +++ /dev/null @@ -1,58 +0,0 @@ -witnesses = [ - { - address = "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" - url = "http://meme5.com" - voteCount = 100000036 - }, - { - address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" - voteCount = 100000035 - }, - { - address = "TKmyxLsRR2FWMVEHaQA2pZh1xB7oXPXzG1" - } -] - -accounts = [ - { - address = "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" - accountName = "Meme" - balance = 99000000000000000 - }, - { - address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" - accountType = "Normal" - balance = 99000000000000000 - }, - { - address = "TLLM21wteSPs4hKjbxgmH1L6poyMjeTbHm" - owner = "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" - }, - { - address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" - trc10Id = "1000001" - trc10Balance = 100000000 - }, - { - address = "TKmyxLsRR2FWMVEHaQA2pZh1xB7oXPXzG1" - } -] - -trc20Contracts = [ - { - contractAddress = "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t" - balancesSlotPosition = 0 - address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" - balance = "98800000000000000" - }, - { - contractAddress = "TSSMHYeV2uE9qYH95DqyoCuNCzEL1NvU3S" - balancesSlotPosition = 0 - address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" - balance = "128745186062510400000000000" - } -] - -latestBlockHeaderTimestamp = 1747986162000 # MilliSecond. Used in init setup, setting to current timestamp reduces debug log output and speeds up initialization. This setting does not impact the underlying block production logic. -maintenanceTimeInterval = 21600000 -nextMaintenanceTime = 1747996162000 # MilliSecond. Set next maintenance time. It could be anytime. If you set an expired time, it will trigger maintenance logic immediately after node startup. diff --git a/tools/toolkit/src/main/resources/logback.xml b/tools/toolkit/src/main/resources/logback.xml index dd2f2538..6c415042 100644 --- a/tools/toolkit/src/main/resources/logback.xml +++ b/tools/toolkit/src/main/resources/logback.xml @@ -52,6 +52,6 @@ - + diff --git a/tools/toolkit/src/main/resources/query.conf b/tools/toolkit/src/main/resources/query.conf deleted file mode 100644 index a1df2aee..00000000 --- a/tools/toolkit/src/main/resources/query.conf +++ /dev/null @@ -1,20 +0,0 @@ -vote = { - allWitnesses = true - witnessList = [ - "TPDa1KRmFpLA42WffiXBwEykef9exHNPKV", - "TXnqPcy9zWHYyCiQ7Kmvt3aCx5jm9Qnq3e", - "TCZvvbn4SCVyNhCAt1L8Kp1qk5rtMiKdBB", - "TKPepDpU2FfagEFRy4eaN2MTevWjpzDNmT", - "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3", - "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" - ] -} - -reward = [ - "TLyqzVGLV1srkB7dToTAEqgDSfPtXRJZYH", - "TJvaAeFb8Lykt9RQcVyyTFN2iDvGMuyD4M", - "TNq7jKkTfFEvGXeqqZR5nk6wFse31wXv8B", - "TWocEBj78sJusnVq7xdXx2MR4LQNTqjkc4", - "TDaQAfV7gZSPzx5FZ23kenPwLjr3AGxnmH", - "TXwrVcWA2BPRMs9gGz5DciT6iwdDKagbpB" -] diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbCopyTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbCopyTest.java index 9e488a59..571fd8f5 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbCopyTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbCopyTest.java @@ -4,14 +4,25 @@ import java.util.UUID; import org.junit.Assert; import org.junit.Test; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; public class DbCopyTest extends DbTest { @Test - public void testRun() { + public void testRunForLevelDB() throws RocksDBException, IOException { + init(DbTool.DbType.LevelDB); String[] args = new String[] { "db", "cp", INPUT_DIRECTORY, - genarateTmpDir()}; + generateTmpDir()}; + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testRunForRocksDB() throws RocksDBException, IOException { + init(DbTool.DbType.RocksDB); + String[] args = new String[] { "db", "cp", INPUT_DIRECTORY, + generateTmpDir()}; Assert.assertEquals(0, cli.execute(args)); } @@ -32,7 +43,7 @@ public void testNotExist() { @Test public void testEmpty() throws IOException { String[] args = new String[] {"db", "cp", temporaryFolder.newFolder().toString(), - genarateTmpDir()}; + generateTmpDir()}; Assert.assertEquals(0, cli.execute(args)); } @@ -46,7 +57,7 @@ public void testDestIsExist() throws IOException { @Test public void testSrcIsFile() throws IOException { String[] args = new String[] {"db", "cp", temporaryFolder.newFile().toString(), - genarateTmpDir()}; + generateTmpDir()}; Assert.assertEquals(403, cli.execute(args)); } diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java deleted file mode 100644 index 1dea60b6..00000000 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java +++ /dev/null @@ -1,206 +0,0 @@ -package org.tron.plugins; - -import static org.tron.plugins.DbFork.getActiveWitness; -import static org.tron.plugins.utils.Constant.*; - -import com.google.common.primitives.Bytes; -import com.google.protobuf.ByteString; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.nio.file.Paths; -import java.util.List; -import java.util.stream.Collectors; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.rocksdb.RocksDBException; -import org.tron.common.utils.ByteArray; -import org.tron.common.utils.Commons; -import org.tron.core.capsule.AccountCapsule; -import org.tron.core.capsule.WitnessCapsule; -import org.tron.plugins.utils.Constant; -import org.tron.plugins.utils.FileUtils; -import org.tron.plugins.utils.db.DBInterface; -import org.tron.plugins.utils.db.DbTool; -import picocli.CommandLine; - -public class DbForkTest { - - private DBInterface witnessStore; - private DBInterface witnessScheduleStore; - private DBInterface accountStore; - private DBInterface dynamicPropertiesStore; - private DBInterface accountAssetStore; - private DBInterface assetIssueV2Store; - private DBInterface contractStore; - private DBInterface storageRowStore; - - @Rule - public final TemporaryFolder folder = new TemporaryFolder(); - private String dbPath; - private String forkPath; - - public void createDir() { - String srcDir = dbPath + File.separator + "database"; - FileUtils.createDirIfNotExists(Paths.get(srcDir, WITNESS_STORE).toString()); - FileUtils.createDirIfNotExists(Paths.get(srcDir, WITNESS_SCHEDULE_STORE).toString()); - FileUtils.createDirIfNotExists(Paths.get(srcDir, ACCOUNT_STORE).toString()); - FileUtils.createDirIfNotExists(Paths.get(srcDir, DYNAMIC_PROPERTY_STORE).toString()); - - FileUtils.createDirIfNotExists(Paths.get(srcDir, ACCOUNT_ASSET).toString()); - FileUtils.createDirIfNotExists(Paths.get(srcDir, ASSET_ISSUE_V2).toString()); - FileUtils.createDirIfNotExists(Paths.get(srcDir, CONTRACT_STORE).toString()); - FileUtils.createDirIfNotExists(Paths.get(srcDir, STORAGE_ROW_STORE).toString()); - - } - - public void init() throws IOException, RocksDBException { - - String srcDir = dbPath + File.separator + "database"; - witnessStore = DbTool.getDB(srcDir, Constant.WITNESS_STORE); - witnessScheduleStore = DbTool.getDB(srcDir, Constant.WITNESS_SCHEDULE_STORE); - accountStore = DbTool.getDB(srcDir, Constant.ACCOUNT_STORE); - dynamicPropertiesStore = DbTool.getDB(srcDir, Constant.DYNAMIC_PROPERTY_STORE); - accountAssetStore = DbTool.getDB(srcDir, Constant.ACCOUNT_ASSET); - assetIssueV2Store = DbTool.getDB(srcDir, Constant.ASSET_ISSUE_V2); - contractStore = DbTool.getDB(srcDir, Constant.CONTRACT_STORE); - storageRowStore = DbTool.getDB(srcDir, Constant.STORAGE_ROW_STORE); - } - - public void close() { - DbTool.close(); - } - - @Test - public void testDbFork() throws IOException, RocksDBException { - dbPath = folder.newFolder().toString(); - forkPath = getConfig("fork.conf"); - createDir(); - - String[] args = new String[]{"-d", - dbPath, "-c", - forkPath}; - CommandLine cli = new CommandLine(new DbFork()); - Assert.assertEquals(0, cli.execute(args)); - - init(); - Config forkConfig; - File file = Paths.get(forkPath).toFile(); - if (file.exists() && file.isFile()) { - forkConfig = ConfigFactory.parseFile(Paths.get(forkPath).toFile()); - } else { - throw new IOException("Fork config file [" + forkPath + "] not exist!"); - } - - if (forkConfig.hasPath(WITNESS_KEY)) { - List witnesses = forkConfig.getConfigList(WITNESS_KEY); - if (witnesses.isEmpty()) { - System.out.println("no witness listed in the config."); - } - witnesses = witnesses.stream() - .filter(c -> c.hasPath(WITNESS_ADDRESS)) - .collect(Collectors.toList()); - if (witnesses.isEmpty()) { - System.out.println("no witness listed in the config."); - } - - List witnessAddresses = witnesses.stream().map( - w -> { - ByteString address = ByteString.copyFrom( - Commons.decodeFromBase58Check(w.getString(WITNESS_ADDRESS))); - return address; - } - ).collect(Collectors.toList()); - Assert.assertArrayEquals(getActiveWitness(witnessAddresses), - witnessScheduleStore.get(ACTIVE_WITNESSES)); - - witnesses.stream().forEach( - w -> { - WitnessCapsule witnessCapsule = new WitnessCapsule(witnessStore.get( - Commons.decodeFromBase58Check(w.getString(WITNESS_ADDRESS)))); - if (w.hasPath(WITNESS_VOTE)) { - Assert.assertEquals(w.getLong(WITNESS_VOTE), witnessCapsule.getVoteCount()); - } - if (w.hasPath(WITNESS_URL)) { - Assert.assertEquals(w.getString(WITNESS_URL), witnessCapsule.getUrl()); - } - } - ); - } - - if (forkConfig.hasPath(ACCOUNTS_KEY)) { - List accounts = forkConfig.getConfigList(ACCOUNTS_KEY); - if (accounts.isEmpty()) { - System.out.println("no account listed in the config."); - } - accounts = accounts.stream() - .filter(c -> c.hasPath(ACCOUNT_ADDRESS)) - .collect(Collectors.toList()); - if (accounts.isEmpty()) { - System.out.println("no account listed in the config."); - } - accounts.stream().forEach( - a -> { - byte[] address = Commons.decodeFromBase58Check(a.getString(ACCOUNT_ADDRESS)); - AccountCapsule account = new AccountCapsule(accountStore.get(address)); - Assert.assertNotNull(account); - if (a.hasPath(ACCOUNT_BALANCE)) { - Assert.assertEquals(a.getLong(ACCOUNT_BALANCE), account.getBalance()); - } - if (a.hasPath(ACCOUNT_NAME)) { - Assert.assertArrayEquals(ByteArray.fromString(a.getString(ACCOUNT_NAME)), - account.getAccountName().toByteArray()); - } - if (a.hasPath(ACCOUNT_TYPE)) { - Assert.assertEquals(a.getString(ACCOUNT_TYPE), account.getType().toString()); - } - if (a.hasPath(ACCOUNT_OWNER)) { - Assert.assertArrayEquals(Commons.decodeFromBase58Check(a.getString(ACCOUNT_OWNER)), - account.getPermissionById(0).getKeys(0).getAddress().toByteArray()); - } - if (a.hasPath(ACCOUNT_TRC10_ID) && a.hasPath(ACCOUNT_TRC10_BALANCE) - && a.getLong(ACCOUNT_TRC10_BALANCE) > 0) { - String trc10Id = a.getString(ACCOUNT_TRC10_ID); - if (assetIssueV2Store.get(ByteArray.fromString(trc10Id)) != null) { - if (account.getAssetOptimized()) { - byte[] k = Bytes.concat(address, ByteArray.fromString(trc10Id)); - byte[] value = accountAssetStore.get(k); - Assert.assertEquals(a.getLong(ACCOUNT_TRC10_BALANCE), ByteArray.toLong(value)); - } else { - long value = account.getAssetMapV2().get(trc10Id); - Assert.assertEquals(a.getLong(ACCOUNT_TRC10_BALANCE), value); - } - } - } - }); - } - - if (forkConfig.hasPath(LATEST_BLOCK_TIMESTAMP)) { - long latestBlockHeaderTimestamp = forkConfig.getLong(LATEST_BLOCK_TIMESTAMP); - Assert.assertEquals(latestBlockHeaderTimestamp, - ByteArray.toLong(dynamicPropertiesStore.get(LATEST_BLOCK_HEADER_TIMESTAMP))); - } - - if (forkConfig.hasPath(MAINTENANCE_INTERVAL)) { - long maintenanceTimeInterval = forkConfig.getLong(MAINTENANCE_INTERVAL); - Assert.assertEquals(maintenanceTimeInterval, - ByteArray.toLong(dynamicPropertiesStore.get(MAINTENANCE_TIME_INTERVAL))); - } - - if (forkConfig.hasPath(NEXT_MAINTENANCE_TIME)) { - long nextMaintenanceTime = forkConfig.getLong(NEXT_MAINTENANCE_TIME); - Assert.assertEquals(nextMaintenanceTime, - ByteArray.toLong(dynamicPropertiesStore.get(MAINTENANCE_TIME))); - } - close(); - } - - private String getConfig(String config) { - URL path = DbForkTest.class.getClassLoader().getResource(config); - return path == null ? null : path.getPath(); - } -} diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbMoveTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbMoveTest.java index c1bc6b47..5b25739f 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbMoveTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbMoveTest.java @@ -1,52 +1,41 @@ package org.tron.plugins; -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; - import java.io.File; import java.io.IOException; import java.net.URL; import java.nio.file.Paths; -import java.util.UUID; import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; +import org.junit.After; import org.junit.Assert; -import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; -import org.tron.plugins.utils.FileUtils; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; @Slf4j public class DbMoveTest { private static final String OUTPUT_DIRECTORY = "output-directory-toolkit"; - private static final String OUTPUT_DIRECTORY_DATABASE = - Paths.get(OUTPUT_DIRECTORY,"ori","database").toString(); - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ACCOUNT = "account"; - private static final String TRANS = "trans"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY_DATABASE, ACCOUNT); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + private static final String ACCOUNT = "account"; + private static final String TRANS = "trans"; - file = new File(OUTPUT_DIRECTORY_DATABASE, MARKET); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); - file = new File(OUTPUT_DIRECTORY_DATABASE, TRANS); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + private void init(DbTool.DbType dbType, String path) throws IOException, RocksDBException { + DbTool.getDB(path, ACCOUNT, dbType).close(); + DbTool.getDB(path, DBUtils.MARKET_PAIR_PRICE_TO_ORDER, dbType).close(); + DbTool.getDB(path, TRANS, dbType).close(); } - @AfterClass - public static void destroy() { + @After + public void destroy() { deleteDir(new File(OUTPUT_DIRECTORY)); } @@ -74,9 +63,23 @@ private static String getConfig(String config) { } @Test - public void testMv() { + public void testMvForLevelDB() throws RocksDBException, IOException { + File database = temporaryFolder.newFolder("database"); + init(DbTool.DbType.LevelDB, Paths.get(database.getPath()).toString()); + String[] args = new String[] {"db", "mv", "-d", + database.getParent(), "-c", + getConfig("config.conf")}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(0, cli.execute(args)); + Assert.assertEquals(2, cli.execute(args)); + } + + @Test + public void testMvForRocksDB() throws RocksDBException, IOException { + File database = temporaryFolder.newFolder("database"); + init(DbTool.DbType.RocksDB, Paths.get(database.getPath()).toString()); String[] args = new String[] {"db", "mv", "-d", - Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + database.getParent(), "-c", getConfig("config.conf")}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(0, cli.execute(args)); @@ -84,9 +87,10 @@ public void testMv() { } @Test - public void testDuplicate() { + public void testDuplicate() throws IOException { + File output = temporaryFolder.newFolder(); String[] args = new String[] {"db", "mv", "-d", - Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + output.getPath(), "-c", getConfig("config-duplicate.conf")}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(2, cli.execute(args)); @@ -107,20 +111,19 @@ public void testDicNotExist() { } @Test - public void testConfNotExist() { + public void testConfNotExist() throws IOException { + File output = temporaryFolder.newFolder(); String[] args = new String[] {"db", "mv", "-d", - Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + output.getPath(), "-c", "config.conf"}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(2, cli.execute(args)); } @Test - public void testEmpty() { - File file = new File(OUTPUT_DIRECTORY_DATABASE + File.separator + UUID.randomUUID()); - file.mkdirs(); - file.deleteOnExit(); - String[] args = new String[] {"db", "mv", "-d", file.toString(), "-c", + public void testEmpty() throws IOException { + File output = temporaryFolder.newFolder(); + String[] args = new String[] {"db", "mv", "-d", output.getPath(), "-c", getConfig("config.conf")}; CommandLine cli = new CommandLine(new Toolkit()); diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbRootTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbRootTest.java index b86688f7..9d032a43 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbRootTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbRootTest.java @@ -9,10 +9,8 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.rocksdb.RocksDBException; -import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.db.DBInterface; import org.tron.plugins.utils.db.DbTool; -import org.tron.plugins.utils.db.LevelDBImpl; import picocli.CommandLine; @Slf4j @@ -27,8 +25,18 @@ public class DbRootTest { private static final String EMPTY_DB = "empty"; private static final String ERROR_DB = "error"; + + @Test + public void testRootForLevelDB() throws RocksDBException, IOException { + testRoot(DbTool.DbType.LevelDB); + } + @Test - public void testRoot() throws IOException, RocksDBException { + public void testRootForRocksDB() throws RocksDBException, IOException { + testRoot(DbTool.DbType.RocksDB); + } + + public void testRoot(DbTool.DbType dbType) throws IOException, RocksDBException { File file = folder.newFolder(); @@ -36,8 +44,8 @@ public void testRoot() throws IOException, RocksDBException { Assert.assertTrue(database.mkdirs()); - try (DBInterface normal = DbTool.getDB(database.toString(), NORMAL_DB, DbTool.DbType.LevelDB); - DBInterface empty = DbTool.getDB(database.toString(), EMPTY_DB, DbTool.DbType.RocksDB)) { + try (DBInterface normal = DbTool.getDB(database.toString(), NORMAL_DB, dbType); + DBInterface empty = DbTool.getDB(database.toString(), EMPTY_DB, dbType)) { for (int i = 0; i < 10; i++) { normal.put(("" + i).getBytes(), (NORMAL_DB + "-" + i).getBytes()); } @@ -53,8 +61,7 @@ public void testRoot() throws IOException, RocksDBException { "--db", EMPTY_DB}; Assert.assertEquals(0, cli.execute(args)); - try (DBInterface errorDb = new LevelDBImpl( - DBUtils.newLevelDb(Paths.get(database.toString(), ERROR_DB)), ERROR_DB)) { + try (DBInterface errorDb = DbTool.getDB(database.toString(), ERROR_DB, dbType)) { for (int i = 0; i < 10; i++) { errorDb.put(("" + i).getBytes(), (ERROR_DB + "-" + i).getBytes()); } diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbTest.java index 8605fa18..bbcc1a0b 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbTest.java @@ -5,42 +5,43 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.UUID; -import org.iq80.leveldb.DB; -import org.junit.Before; +import org.junit.Assert; import org.junit.Rule; import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; import org.tron.plugins.utils.ByteArray; import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.MarketUtils; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; public class DbTest { - String INPUT_DIRECTORY; + public String INPUT_DIRECTORY; private static final String ACCOUNT = "account"; private static final String MARKET = DBUtils.MARKET_PAIR_PRICE_TO_ORDER; - CommandLine cli = new CommandLine(new Toolkit()); - String tmpDir = System.getProperty("java.io.tmpdir"); + public CommandLine cli = new CommandLine(new Toolkit()); @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); - @Before - public void init() throws IOException { + public void init(DbTool.DbType dbType) throws IOException, RocksDBException { INPUT_DIRECTORY = temporaryFolder.newFolder().toString(); - initDB(new File(INPUT_DIRECTORY, ACCOUNT)); - initDB(new File(INPUT_DIRECTORY, MARKET)); - initDB(new File(INPUT_DIRECTORY, DBUtils.CHECKPOINT_DB_V2)); + initDB(INPUT_DIRECTORY, ACCOUNT, dbType); + initDB(INPUT_DIRECTORY, MARKET, dbType); + initDB(INPUT_DIRECTORY, DBUtils.CHECKPOINT_DB_V2, dbType); } - private static void initDB(File file) throws IOException { - if (DBUtils.CHECKPOINT_DB_V2.equalsIgnoreCase(file.getName())) { - File dbFile = new File(file, DBUtils.CHECKPOINT_DB_V2); + private static void initDB(String sourceDir, String dbName, DbTool.DbType dbType) + throws IOException, RocksDBException { + if (DBUtils.CHECKPOINT_DB_V2.equalsIgnoreCase(dbName)) { + File dbFile = new File(sourceDir, DBUtils.CHECKPOINT_DB_V2); if (dbFile.mkdirs()) { for (int i = 0; i < 3; i++) { - try (DB db = DBUtils.newLevelDb(Paths.get(dbFile.getPath(), - System.currentTimeMillis() + ""))) { + try (DBInterface db = DbTool.getDB(dbFile.getPath(), + System.currentTimeMillis() + "", dbType)) { for (int j = 0; j < 100; j++) { byte[] bytes = UUID.randomUUID().toString().getBytes(); db.put(bytes, bytes); @@ -50,8 +51,8 @@ private static void initDB(File file) throws IOException { } return; } - try (DB db = DBUtils.newLevelDb(file.toPath())) { - if (MARKET.equalsIgnoreCase(file.getName())) { + try (DBInterface db = DbTool.getDB(sourceDir, dbName, dbType)) { + if (MARKET.equalsIgnoreCase(dbName)) { byte[] sellTokenID1 = ByteArray.fromString("100"); byte[] buyTokenID1 = ByteArray.fromString("200"); byte[] pairPriceKey1 = MarketUtils.createPairPriceKey( @@ -73,26 +74,29 @@ private static void initDB(File file) throws IOException { 2003L ); - //Use out-of-order insertion，key in store should be 1,2,3 db.put(pairPriceKey1, "1".getBytes(StandardCharsets.UTF_8)); db.put(pairPriceKey2, "2".getBytes(StandardCharsets.UTF_8)); db.put(pairPriceKey3, "3".getBytes(StandardCharsets.UTF_8)); + Assert.assertEquals(3, db.size()); } else { for (int i = 0; i < 100; i++) { byte[] bytes = UUID.randomUUID().toString().getBytes(); db.put(bytes, bytes); } + Assert.assertEquals(100, db.size()); } } } /** * Generate a not-exist temporary directory path. + * * @return temporary path */ - public String genarateTmpDir() { - File dir = Paths.get(tmpDir, UUID.randomUUID().toString()).toFile(); + public String generateTmpDir() throws IOException { + File dir = Paths.get(temporaryFolder.newFolder().toString(), UUID.randomUUID().toString()) + .toFile(); dir.deleteOnExit(); return dir.getPath(); } diff --git a/tools/toolkit/src/test/java/org/tron/plugins/ArchiveManifestTest.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/ArchiveManifestTest.java similarity index 69% rename from tools/toolkit/src/test/java/org/tron/plugins/ArchiveManifestTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/ArchiveManifestTest.java index 4fd9e537..f5880d82 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/ArchiveManifestTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/ArchiveManifestTest.java @@ -1,6 +1,4 @@ -package org.tron.plugins; - -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; +package org.tron.plugins.leveldb; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -14,47 +12,41 @@ import java.nio.charset.StandardCharsets; import java.util.Properties; import java.util.UUID; - import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.ArchiveManifest; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DbTool; @Slf4j public class ArchiveManifestTest { - private static final String OUTPUT_DIRECTORY = "output-directory/database/archiveManifest"; + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static String OUTPUT_DIRECTORY; - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ROCKSDB = "ROCKSDB"; private static final String ACCOUNT = "account"; private static final String ACCOUNT_ROCKSDB = "account-rocksdb"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; - @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY,ACCOUNT); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); - - file = new File(OUTPUT_DIRECTORY,MARKET); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + public static void init() throws IOException, RocksDBException { + OUTPUT_DIRECTORY = temporaryFolder.newFolder("database").toString(); + File file = new File(OUTPUT_DIRECTORY, ACCOUNT); + DbTool.openLevelDb(file.toPath(),ACCOUNT).close(); - file = new File(OUTPUT_DIRECTORY,ACCOUNT_ROCKSDB); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,ROCKSDB); + file = new File(OUTPUT_DIRECTORY, DBUtils.MARKET_PAIR_PRICE_TO_ORDER); + DbTool.openLevelDb(file.toPath(), DBUtils.MARKET_PAIR_PRICE_TO_ORDER).close(); - } + file = new File(OUTPUT_DIRECTORY, ACCOUNT_ROCKSDB); + DbTool.openRocksDb(file.toPath(), ACCOUNT_ROCKSDB).close(); - @AfterClass - public static void destroy() { - deleteDir(new File(OUTPUT_DIRECTORY)); } @Test diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbArchiveTest.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbArchiveTest.java similarity index 71% rename from tools/toolkit/src/test/java/org/tron/plugins/DbArchiveTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbArchiveTest.java index 10bed418..69dca01e 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbArchiveTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbArchiveTest.java @@ -1,6 +1,4 @@ -package org.tron.plugins; - -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; +package org.tron.plugins.leveldb; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -12,48 +10,43 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; import java.util.Properties; import java.util.UUID; import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.Toolkit; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; @Slf4j public class DbArchiveTest { - private static final String OUTPUT_DIRECTORY = "output-directory/database/dbArchive"; + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static String OUTPUT_DIRECTORY; - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ROCKSDB = "ROCKSDB"; private static final String ACCOUNT = "account"; private static final String ACCOUNT_ROCKSDB = "account-rocksdb"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY,ACCOUNT); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + public static void init() throws IOException, RocksDBException { + OUTPUT_DIRECTORY = temporaryFolder.newFolder("database").toString(); + File file = new File(OUTPUT_DIRECTORY, ACCOUNT); + DbTool.openLevelDb(file.toPath(),ACCOUNT).close(); - file = new File(OUTPUT_DIRECTORY,MARKET); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + file = new File(OUTPUT_DIRECTORY, DBUtils.MARKET_PAIR_PRICE_TO_ORDER); + DbTool.openLevelDb(file.toPath(), DBUtils.MARKET_PAIR_PRICE_TO_ORDER).close(); - file = new File(OUTPUT_DIRECTORY,ACCOUNT_ROCKSDB); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,ROCKSDB); - - } + file = new File(OUTPUT_DIRECTORY, ACCOUNT_ROCKSDB); + DbTool.openRocksDb(file.toPath(), ACCOUNT_ROCKSDB).close(); - @AfterClass - public static void destroy() { - deleteDir(new File(OUTPUT_DIRECTORY)); } @Test diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbConvertTest.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbConvertTest.java similarity index 76% rename from tools/toolkit/src/test/java/org/tron/plugins/DbConvertTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbConvertTest.java index 150e47c9..d24604f0 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbConvertTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbConvertTest.java @@ -1,27 +1,25 @@ -package org.tron.plugins; +package org.tron.plugins.leveldb; import java.io.IOException; import java.util.UUID; import org.junit.Assert; import org.junit.Test; +import org.rocksdb.RocksDBException; +import org.tron.plugins.DbTest; +import org.tron.plugins.Toolkit; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; public class DbConvertTest extends DbTest { @Test - public void testRun() throws IOException { + public void testRun() throws IOException, RocksDBException { + init(DbTool.DbType.LevelDB); String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, temporaryFolder.newFolder().toString() }; Assert.assertEquals(0, cli.execute(args)); } - @Test - public void testRunWithSafe() throws IOException { - String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, - temporaryFolder.newFolder().toString(),"--safe" }; - Assert.assertEquals(0, cli.execute(args)); - } - @Test public void testHelp() { String[] args = new String[] {"db", "convert", "-h"}; diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java similarity index 76% rename from tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java index 792f93ad..7666806e 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java @@ -1,7 +1,8 @@ -package org.tron.plugins; +package org.tron.plugins.leveldb; import java.io.IOException; import org.junit.Test; +import org.tron.plugins.DbLiteTest; public class DbLiteLevelDbTest extends DbLiteTest { diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java similarity index 76% rename from tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java index ae48e1d6..de32ae29 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java @@ -1,7 +1,8 @@ -package org.tron.plugins; +package org.tron.plugins.leveldb; import java.io.IOException; import org.junit.Test; +import org.tron.plugins.DbLiteTest; public class DbLiteLevelDbV2Test extends DbLiteTest { diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java b/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java similarity index 76% rename from tools/toolkit/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java index e6910b11..2f9c92f9 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java @@ -1,7 +1,8 @@ -package org.tron.plugins; +package org.tron.plugins.rocksdb; import java.io.IOException; import org.junit.Test; +import org.tron.plugins.DbLiteTest; public class DbLiteRocksDbTest extends DbLiteTest { diff --git a/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java b/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java new file mode 100644 index 00000000..ab1067fe --- /dev/null +++ b/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java @@ -0,0 +1,13 @@ +package org.tron.plugins.rocksdb; + +import java.io.IOException; +import org.junit.Test; +import org.tron.plugins.DbLiteTest; + +public class DbLiteRocksDbV2Test extends DbLiteTest { + + @Test + public void testToolsWithRocksDB() throws InterruptedException, IOException { + testTools("ROCKSDB", 2); + } +} diff --git a/tools/toolkit/src/test/resources/config-duplicate.conf b/tools/toolkit/src/test/resources/config-duplicate.conf index 05b7b9f4..f2eb7fbf 100644 --- a/tools/toolkit/src/test/resources/config-duplicate.conf +++ b/tools/toolkit/src/test/resources/config-duplicate.conf @@ -17,3 +17,5 @@ storage { }, ] } + + diff --git a/tools/toolkit/src/test/resources/config-localtest.conf b/tools/toolkit/src/test/resources/config-localtest.conf index b280997c..8049ceb6 100644 --- a/tools/toolkit/src/test/resources/config-localtest.conf +++ b/tools/toolkit/src/test/resources/config-localtest.conf @@ -90,7 +90,7 @@ node { minParticipationRate = 0 - fullNodeAllowShieldedTransaction = true + allowShieldedTransactionApi = true zenTokenId = 1000001 @@ -117,14 +117,21 @@ node { ] http { + fullNodeEnable = false fullNodePort = 8090 + solidityEnable = false solidityPort = 8091 + PBFTEnable = false + PBFTPort = 8092 } rpc { + enable = false port = 50051 - # default value is 50061 - # solidityPort = 50061 + solidityEnable = false + solidityPort = 50061 + PBFTEnable = false + PBFTPort = 50071 # Number of gRPC thread, default availableProcessors / 2 # thread = 16 @@ -158,6 +165,9 @@ node { # httpSolidityPort = 8555 # httpPBFTEnable = true # httpPBFTPort = 8565 + # maxBlockRange = 5000 + # maxSubTopics = 1000 + # maxBlockFilterNum = 30000 } } @@ -209,7 +219,7 @@ genesis.block = { witnesses = [ { - address: "TN3zfjYUmMFK3ZsHSsrdJoNRtGkQmZLBLz" + address: TEDapYSVvAZ3aYH7w8N9tMEEFKaNKUD5Bp url = "http://Test.org", voteCount = 106 }, @@ -246,7 +256,7 @@ genesis.block = { //localWitnessAccountAddress = TN3zfjYUmMFK3ZsHSsrdJoNRtGkQmZLBLz localwitness = [ - f4df789d3210ac881cb900464dd30409453044d2777060a0c391cbdf4c6a4f57 + 1234567890123456789012345678901234567890123456789012345678901234 ] diff --git a/tools/toolkit/src/test/resources/config.conf b/tools/toolkit/src/test/resources/config.conf index a596d68e..77d15d52 100644 --- a/tools/toolkit/src/test/resources/config.conf +++ b/tools/toolkit/src/test/resources/config.conf @@ -1,11 +1,5 @@ storage { - # Directory for storing persistent data - db.engine = "LEVELDB", - db.sync = false, - db.directory = "database", - index.directory = "index", - transHistory.switch = "on", properties = [ { name = "account", @@ -17,3 +11,5 @@ storage { }, ] } + + diff --git a/tools/toolkit/src/test/resources/fork.conf b/tools/toolkit/src/test/resources/fork.conf deleted file mode 100644 index 602eb4d6..00000000 --- a/tools/toolkit/src/test/resources/fork.conf +++ /dev/null @@ -1,58 +0,0 @@ -witnesses = [ - { - address = "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" - url = "http://meme5.com" - voteCount = 100000036 - }, - { - address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" - voteCount = 100000035 - }, - { - address = "TKmyxLsRR2FWMVEHaQA2pZh1xB7oXPXzG1" - } -] - -accounts = [ - { - address = "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" - accountName = "Meme" - balance = 99000000000000000 - }, - { - address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" - accountType = "Normal" - balance = 99000000000000000 - }, - { - address = "TLLM21wteSPs4hKjbxgmH1L6poyMjeTbHm" - owner = "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" - }, - { - address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" - trc10Id = "1000001" - trc10Balance = 100000000 - }, - { - address = "TKmyxLsRR2FWMVEHaQA2pZh1xB7oXPXzG1" - } -] - -trc20Contracts = [ - { - contractAddress = "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t" - balancesSlotPosition = 0 - address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" - balance = "98800000000000000" - }, - { - contractAddress = "TSSMHYeV2uE9qYH95DqyoCuNCzEL1NvU3S" - balancesSlotPosition = 0 - address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" - balance = "128745186062510400000000000" - } -] - -latestBlockHeaderTimestamp = 1739269661000 -maintenanceTimeInterval = 21600000 -nextMaintenanceTime = 1735628894000 From 28cbf7b745ba2d1a99f7c485ac87882ba2d1fb0f Mon Sep 17 00:00:00 2001 From: federico Date: Mon, 26 Jan 2026 14:28:45 +0800 Subject: [PATCH 02/11] fix the error --- .../java/org/tron/plugins/DbLiteTest.java | 48 ++++++++----------- .../org/tron/plugins/utils/PublicMethod.java | 26 ++++++---- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbLiteTest.java index 3d3b9d5f..1d5771a0 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbLiteTest.java @@ -1,12 +1,12 @@ package org.tron.plugins; +import static org.tron.plugins.utils.PublicMethod.getRandomPrivateKey; + import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import java.io.File; import java.io.IOException; -import java.net.URL; import java.nio.file.Paths; -import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Rule; @@ -18,11 +18,10 @@ import org.tron.common.config.DbBackupConfig; import org.tron.common.crypto.ECKey; import org.tron.common.utils.FileUtil; +import org.tron.common.utils.TimeoutInterceptor; import org.tron.common.utils.Utils; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; -import org.tron.core.services.RpcApiService; -import org.tron.core.services.interfaceOnSolidity.RpcApiServiceOnSolidity; import org.tron.plugins.utils.PublicMethod; import picocli.CommandLine; @@ -47,14 +46,13 @@ public class DbLiteTest { public void startApp() { context = new TronApplicationContext(DefaultConfig.class); appTest = ApplicationFactory.create(context); - appTest.addService(context.getBean(RpcApiService.class)); - appTest.addService(context.getBean(RpcApiServiceOnSolidity.class)); appTest.startup(); String fullNode = String.format("%s:%d", "127.0.0.1", Args.getInstance().getRpcPort()); channelFull = ManagedChannelBuilder.forTarget(fullNode) .usePlaintext() + .intercept(new TimeoutInterceptor(5000)) .build(); blockingStubFull = WalletGrpc.newBlockingStub(channelFull); } @@ -64,18 +62,20 @@ public void startApp() { */ public void shutdown() throws InterruptedException { if (channelFull != null) { - channelFull.shutdown().awaitTermination(5, TimeUnit.SECONDS); + channelFull.shutdownNow(); } context.close(); } - public void init() throws IOException { + public void init(String dbType) throws IOException { dbPath = folder.newFolder().toString(); - Args.setParam(new String[]{"-d", dbPath, "-w", "--p2p-disable", "true"}, - getConfig("config-localtest.conf")); + Args.setParam(new String[] { + "-d", dbPath, "-w", "--p2p-disable", "true", "--storage-db-engine", dbType}, + "config-localtest.conf"); // allow account root Args.getInstance().setAllowAccountStateRoot(1); Args.getInstance().setRpcPort(PublicMethod.chooseRandomPort()); + Args.getInstance().setRpcEnable(true); databaseDir = Args.getInstance().getStorage().getDbDirectory(); // init dbBackupConfig to avoid NPE Args.getInstance().dbBackupConfig = DbBackupConfig.getInstance(); @@ -86,23 +86,22 @@ public void clear() { Args.clearParam(); } - void testTools(String dbType, int checkpointVersion) + public void testTools(String dbType, int checkpointVersion) throws InterruptedException, IOException { logger.info("dbType {}, checkpointVersion {}", dbType, checkpointVersion); dbPath = String.format("%s_%s_%d", dbPath, dbType, System.currentTimeMillis()); - init(); + init(dbType); final String[] argsForSnapshot = - new String[]{"-o", "split", "-t", "snapshot", "--fn-data-path", + new String[] {"-o", "split", "-t", "snapshot", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath}; final String[] argsForHistory = - new String[]{"-o", "split", "-t", "history", "--fn-data-path", + new String[] {"-o", "split", "-t", "history", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath}; final String[] argsForMerge = - new String[]{"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, + new String[] {"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath + File.separator + "history"}; - Args.getInstance().getStorage().setDbEngine(dbType); Args.getInstance().getStorage().setCheckpointVersion(checkpointVersion); DbLite.setRecentBlks(3); // start fullNode @@ -127,15 +126,15 @@ void testTools(String dbType, int checkpointVersion) File database = new File(Paths.get(dbPath, databaseDir).toString()); if (!database.renameTo(new File(Paths.get(dbPath, databaseDir + "_bak").toString()))) { throw new RuntimeException( - String.format("rename %s to %s failed", database.getPath(), - Paths.get(dbPath, databaseDir))); + String.format("rename %s to %s failed", database.getPath(), + Paths.get(dbPath, databaseDir))); } // change snapshot to the new database File snapshot = new File(Paths.get(dbPath, "snapshot").toString()); if (!snapshot.renameTo(new File(Paths.get(dbPath, databaseDir).toString()))) { throw new RuntimeException( - String.format("rename snapshot to %s failed", - Paths.get(dbPath, databaseDir))); + String.format("rename snapshot to %s failed", + Paths.get(dbPath, databaseDir))); } // start and validate the snapshot startApp(); @@ -159,10 +158,10 @@ private void generateSomeTransactions(int during) { ECKey ecKey2 = new ECKey(Utils.getRandom()); byte[] address = ecKey2.getAddress(); - String sunPri = "cba92a516ea09f620a16ff7ee95ce0df1d56550a8babe9964981a7144c8a784a"; + String sunPri = getRandomPrivateKey(); byte[] sunAddress = PublicMethod.getFinalAddress(sunPri); PublicMethod.sendcoin(address, 1L, - sunAddress, sunPri, blockingStubFull); + sunAddress, sunPri, blockingStubFull); try { Thread.sleep(sleepOnce); } catch (InterruptedException e) { @@ -173,9 +172,4 @@ private void generateSomeTransactions(int during) { } } } - - private static String getConfig(String config) { - URL path = DbLiteTest.class.getClassLoader().getResource(config); - return path == null ? null : path.getPath(); - } } diff --git a/tools/toolkit/src/test/java/org/tron/plugins/utils/PublicMethod.java b/tools/toolkit/src/test/java/org/tron/plugins/utils/PublicMethod.java index ce191c83..5e21a693 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/utils/PublicMethod.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/utils/PublicMethod.java @@ -6,22 +6,28 @@ import java.net.InetAddress; import java.net.Socket; import java.util.List; +import java.util.Objects; import java.util.Random; +import org.bouncycastle.util.encoders.Hex; import org.tron.api.GrpcAPI; import org.tron.api.WalletGrpc; import org.tron.common.crypto.ECKey; import org.tron.common.crypto.ECKey.ECDSASignature; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.Sha256Hash; +import org.tron.common.utils.Utils; import org.tron.core.Wallet; -import org.tron.protos.Protocol; import org.tron.protos.Protocol.Transaction; import org.tron.protos.Protocol.Transaction.Contract; -import org.tron.protos.contract.BalanceContract; import org.tron.protos.contract.BalanceContract.TransferContract; public class PublicMethod { + public static String getRandomPrivateKey() { + return Hex.toHexString(Objects + .requireNonNull(new ECKey(Utils.getRandom()).getPrivKeyBytes())); + } + /** * Transfer TRX. * @param to addr receives the asset @@ -47,15 +53,15 @@ public static Boolean sendcoin(byte[] to, long amount, byte[] owner, String priK while (times++ <= 2) { TransferContract.Builder builder = - BalanceContract.TransferContract.newBuilder(); - com.google.protobuf.ByteString bsTo = com.google.protobuf.ByteString.copyFrom(to); - com.google.protobuf.ByteString bsOwner = ByteString.copyFrom(owner); + TransferContract.newBuilder(); + ByteString bsTo = ByteString.copyFrom(to); + ByteString bsOwner = ByteString.copyFrom(owner); builder.setToAddress(bsTo); builder.setOwnerAddress(bsOwner); builder.setAmount(amount); TransferContract contract = builder.build(); - Protocol.Transaction transaction = blockingStubFull.createTransaction(contract); + Transaction transaction = blockingStubFull.createTransaction(contract); if (transaction == null || transaction.getRawData().getContractCount() == 0) { continue; } @@ -71,8 +77,8 @@ public static Boolean sendcoin(byte[] to, long amount, byte[] owner, String priK * @param ecKey ecKey of the private key * @param transaction transaction object */ - public static Protocol.Transaction signTransaction(ECKey ecKey, - Protocol.Transaction transaction) { + public static Transaction signTransaction(ECKey ecKey, + Transaction transaction) { if (ecKey == null || ecKey.getPrivKey() == null) { return null; } @@ -83,7 +89,7 @@ public static Protocol.Transaction signTransaction(ECKey ecKey, public static Transaction setTimestamp(Transaction transaction) { long currentTime = System.currentTimeMillis();//*1000000 + System.nanoTime()%1000000; Transaction.Builder builder = transaction.toBuilder(); - org.tron.protos.Protocol.Transaction.raw.Builder rowBuilder = transaction.getRawData() + Transaction.raw.Builder rowBuilder = transaction.getRawData() .toBuilder(); rowBuilder.setTimestamp(currentTime); builder.setRawData(rowBuilder.build()); @@ -113,7 +119,7 @@ public static Transaction sign(Transaction transaction, ECKey myKey) { * @param blockingStubFull Grpc interface */ public static GrpcAPI.Return broadcastTransaction( - Protocol.Transaction transaction, WalletGrpc.WalletBlockingStub blockingStubFull) { + Transaction transaction, WalletGrpc.WalletBlockingStub blockingStubFull) { int i = 10; GrpcAPI.Return response = blockingStubFull.broadcastTransaction(transaction); while (!response.getResult() && response.getCode() == GrpcAPI.Return.response_code.SERVER_BUSY From ad2f03c1bc3074e32473790983c52f9f5babcf71 Mon Sep 17 00:00:00 2001 From: federico Date: Mon, 26 Jan 2026 15:40:42 +0800 Subject: [PATCH 03/11] add the DbFork and DbQuery --- tools/toolkit/README.md | 18 +- .../main/java/common/org/tron/plugins/Db.java | 4 +- .../java/common/org/tron/plugins/DbFork.java | 419 +++++ .../java/common/org/tron/plugins/DbQuery.java | 702 +++++++ .../org/tron/plugins/utils/Constant.java | 63 + .../utils/HttpSelfFormatFieldName.java | 283 +++ .../org/tron/plugins/utils/JsonFormat.java | 1663 +++++++++++++++++ tools/toolkit/src/main/resources/fork.conf | 58 + tools/toolkit/src/main/resources/logback.xml | 3 +- tools/toolkit/src/main/resources/query.conf | 20 + .../java/org/tron/plugins/DbForkTest.java | 227 +++ tools/toolkit/src/test/resources/fork.conf | 58 + 12 files changed, 3505 insertions(+), 13 deletions(-) create mode 100644 tools/toolkit/src/main/java/common/org/tron/plugins/DbFork.java create mode 100644 tools/toolkit/src/main/java/common/org/tron/plugins/DbQuery.java create mode 100644 tools/toolkit/src/main/java/common/org/tron/plugins/utils/Constant.java create mode 100644 tools/toolkit/src/main/java/common/org/tron/plugins/utils/HttpSelfFormatFieldName.java create mode 100644 tools/toolkit/src/main/java/common/org/tron/plugins/utils/JsonFormat.java create mode 100644 tools/toolkit/src/main/resources/fork.conf create mode 100644 tools/toolkit/src/main/resources/query.conf create mode 100644 tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java create mode 100644 tools/toolkit/src/test/resources/fork.conf diff --git a/tools/toolkit/README.md b/tools/toolkit/README.md index 082fb589..3dcd4c4d 100644 --- a/tools/toolkit/README.md +++ b/tools/toolkit/README.md @@ -23,11 +23,11 @@ expenses. - `convert`: Covert leveldb to rocksdb. - `lite`: Split lite data for java-tron. - `cp, copy`: Quick copy leveldb or rocksdb data. -- `root`: compute merkle root for tiny db. NOTE: large db may GC overhead -limit exceeded. +- `root`: compute merkle root for tiny db. NOTE: large db may GC overhead limit exceeded. - `fork`: Modify the database of java-tron for shadow fork testing. +- `query`: Query the latest vote and reward information from the database. -## DB Archive +## DB Archive(Requires x86 + LevelDB) DB archive provides the ability to reformat the manifest according to the current `database`, parameters are compatible with the previous `ArchiveManifest`. @@ -50,8 +50,7 @@ DB archive provides the ability to reformat the manifest according to the curren java -jar Toolkit.jar db archive -m 128 #4. specify optimization only when Manifest exceeds 128M ``` - -## DB Convert +## DB Convert(Requires x86 + LevelDB) DB convert provides a helper which can convert LevelDB data to RocksDB data, parameters are compatible with previous `DBConvert`. @@ -59,15 +58,13 @@ DB convert provides a helper which can convert LevelDB data to RocksDB data, par - ``: Input path for leveldb, default: output-directory/database. - ``: Output path for rocksdb, default: output-directory-dst/database. -- `--safe`: In safe mode, read data from leveldb then put into rocksdb, it's a very time-consuming procedure. If not, just change engine.properties from leveldb to rocksdb, rocksdb - is compatible with leveldb for the current version. This may not be the case in the future, default: false. - `-h | --help`: Provide the help info. ### Examples: ```shell script # full command - java -jar Toolkit.jar db convert [-h] [--safe] + java -jar Toolkit.jar db convert [-h] # examples java -jar Toolkit.jar db convert output-directory/database /tmp/database ``` @@ -91,7 +88,7 @@ DB copy provides a helper which can copy LevelDB or RocksDB data quickly on the java -jar Toolkit.jar db cp output-directory/database /tmp/databse ``` -## DB Lite +## DB Lite(LevelDB unavailable on ARM) DB lite provides lite database, parameters are compatible with previous `LiteFullNodeTool`. @@ -159,7 +156,7 @@ Execute move command. java -jar Toolkit.jar db mv -c main_net_config.conf -d /data/tron/output-directory ``` -## DB Root +## DB Root(LevelDB unavailable on ARM) DB root provides a helper which can compute merkle root for tiny db. @@ -171,7 +168,6 @@ NOTE: large db may GC overhead limit exceeded. - `--db`: db name. - `-h | --help`: provide the help info - ## DB Fork DB fork tool can help launch a private java-tron FullNode or network based on the state of public chain database to support shadow fork testing. diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/Db.java b/tools/toolkit/src/main/java/common/org/tron/plugins/Db.java index 84654dca..298718a9 100644 --- a/tools/toolkit/src/main/java/common/org/tron/plugins/Db.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/Db.java @@ -12,7 +12,9 @@ DbConvert.class, DbLite.class, DbCopy.class, - DbRoot.class + DbRoot.class, + DbFork.class, + DbQuery.class }, commandListHeading = "%nCommands:%n%nThe most commonly used db commands are:%n" ) diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/DbFork.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbFork.java new file mode 100644 index 00000000..0e6919f7 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbFork.java @@ -0,0 +1,419 @@ +package org.tron.plugins; + +import static java.lang.System.arraycopy; +import static org.tron.plugins.utils.Constant.ACCOUNTS_KEY; +import static org.tron.plugins.utils.Constant.ACCOUNT_ADDRESS; +import static org.tron.plugins.utils.Constant.ACCOUNT_ASSET; +import static org.tron.plugins.utils.Constant.ACCOUNT_BALANCE; +import static org.tron.plugins.utils.Constant.ACCOUNT_NAME; +import static org.tron.plugins.utils.Constant.ACCOUNT_OWNER; +import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; +import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_BALANCE; +import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_ID; +import static org.tron.plugins.utils.Constant.ACCOUNT_TYPE; +import static org.tron.plugins.utils.Constant.ACTIVE_WITNESSES; +import static org.tron.plugins.utils.Constant.ASSET_ISSUE_V2; +import static org.tron.plugins.utils.Constant.CONTRACT_STORE; +import static org.tron.plugins.utils.Constant.DYNAMIC_PROPERTY_STORE; +import static org.tron.plugins.utils.Constant.LATEST_BLOCK_HEADER_TIMESTAMP; +import static org.tron.plugins.utils.Constant.LATEST_BLOCK_TIMESTAMP; +import static org.tron.plugins.utils.Constant.MAINTENANCE_INTERVAL; +import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME; +import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME_INTERVAL; +import static org.tron.plugins.utils.Constant.MAX_ACTIVE_WITNESS_NUM; +import static org.tron.plugins.utils.Constant.NEXT_MAINTENANCE_TIME; +import static org.tron.plugins.utils.Constant.STORAGE_ROW_STORE; +import static org.tron.plugins.utils.Constant.TRC20_ACCOUNT; +import static org.tron.plugins.utils.Constant.TRC20_BALANCE; +import static org.tron.plugins.utils.Constant.TRC20_BALANCES_POSITION; +import static org.tron.plugins.utils.Constant.TRC20_CONTRACTS_KEY; +import static org.tron.plugins.utils.Constant.TRC20_CONTRACT_ADDRESS; +import static org.tron.plugins.utils.Constant.WITNESS_ADDRESS; +import static org.tron.plugins.utils.Constant.WITNESS_KEY; +import static org.tron.plugins.utils.Constant.WITNESS_SCHEDULE_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_URL; +import static org.tron.plugins.utils.Constant.WITNESS_VOTE; + +import com.google.common.primitives.Bytes; +import com.google.common.primitives.Longs; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; +import org.rocksdb.RocksDBException; +import org.tron.common.crypto.Hash; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.ByteUtil; +import org.tron.common.utils.Commons; +import org.tron.core.capsule.AccountCapsule; +import org.tron.core.capsule.StorageRowCapsule; +import org.tron.core.capsule.WitnessCapsule; +import org.tron.plugins.utils.Constant; +import org.tron.plugins.utils.FileUtils; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DBIterator; +import org.tron.plugins.utils.db.DbTool; +import org.tron.protos.Protocol.Account; +import org.tron.protos.Protocol.AccountType; +import org.tron.protos.Protocol.Permission; +import org.tron.protos.contract.SmartContractOuterClass.SmartContract; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +@Slf4j(topic = "fork") +@Command(name = "fork", + description = "Modify the database of java-tron for shadow fork testing.", + exitCodeListHeading = "Exit Codes:%n", + exitCodeList = { + "0:Successful", + "n:Internal error: exception occurred, please check logs/toolkit.log"}) +public class DbFork implements Callable { + + private DBInterface witnessStore; + private DBInterface witnessScheduleStore; + private DBInterface accountStore; + private DBInterface dynamicPropertiesStore; + private DBInterface accountAssetStore; + private DBInterface assetIssueV2Store; + private DBInterface contractStore; + private DBInterface storageRowStore; + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + + @CommandLine.Option(names = {"-d", "--database-directory"}, + defaultValue = "output-directory", + description = "java-tron database directory path. Default: ${DEFAULT-VALUE}") + private String database; + + @CommandLine.Option(names = {"-c", "--config"}, + defaultValue = "fork.conf", + description = "config the new witnesses, balances, etc for shadow fork." + + " Default: ${DEFAULT-VALUE}") + private String config; + + @CommandLine.Option(names = {"-r", "--retain-witnesses"}, + description = "retain the previous witnesses and active witnesses.") + private boolean retain; + + @CommandLine.Option(names = {"-h", "--help"}) + private boolean help; + + private void initStore() throws IOException, RocksDBException { + String srcDir = database + File.separator + "database"; + witnessStore = DbTool.getDB(srcDir, WITNESS_STORE); + witnessScheduleStore = DbTool.getDB(srcDir, WITNESS_SCHEDULE_STORE); + accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); + dynamicPropertiesStore = DbTool.getDB(srcDir, DYNAMIC_PROPERTY_STORE); + accountAssetStore = DbTool.getDB(srcDir, ACCOUNT_ASSET); + assetIssueV2Store = DbTool.getDB(srcDir, ASSET_ISSUE_V2); + contractStore = DbTool.getDB(srcDir, CONTRACT_STORE); + storageRowStore = DbTool.getDB(srcDir, STORAGE_ROW_STORE); + } + + @Override + public Integer call() throws IOException, RocksDBException { + if (help) { + spec.commandLine().usage(System.out); + return 0; + } + + File dbFile = Paths.get(database).toFile(); + if (!dbFile.exists() || !dbFile.isDirectory()) { + logger.error("Database [" + database + "] not exists!"); + spec.commandLine().getErr().format("Database %s not exists!", database).println(); + System.exit(-1); + } + File tmp = Paths.get(database, "database", "tmp").toFile(); + if (tmp.exists()) { + FileUtils.deleteDir(tmp); + } + + Config forkConfig = ConfigFactory.load(); + File file = Paths.get(config).toFile(); + if (file.exists() && file.isFile()) { + forkConfig = ConfigFactory.parseFile(file); + } else { + logger.error("Fork config file [" + config + "] not exists!"); + spec.commandLine().getErr().format("Fork config file: %s not exists!", config).println(); + System.exit(-1); + } + + initStore(); + + if (!retain) { + logger.info("Erase the previous witnesses and active witnesses."); + spec.commandLine().getOut().println("Erase the previous witnesses and active witnesses."); + witnessScheduleStore.delete(ACTIVE_WITNESSES); + DBIterator iterator = witnessStore.iterator(); + for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { + witnessStore.delete(iterator.getKey()); + } + } else { + logger.warn("Keep the previous witnesses and active witnesses."); + spec.commandLine().getOut().println("Keep the previous witnesses and active witnesses."); + } + + if (forkConfig.hasPath(WITNESS_KEY)) { + List witnesses = forkConfig.getConfigList(WITNESS_KEY); + if (witnesses.isEmpty()) { + spec.commandLine().getOut().println("no witness listed in the config."); + } + witnesses = witnesses.stream() + .filter(c -> c.hasPath(WITNESS_ADDRESS)) + .collect(Collectors.toList()); + + if (witnesses.isEmpty()) { + spec.commandLine().getOut().println("no witness listed in the config."); + } + + List witnessList = new ArrayList<>(); + witnesses.stream().forEach( + w -> { + ByteString address = ByteString.copyFrom( + Commons.decodeFromBase58Check(w.getString(WITNESS_ADDRESS))); + WitnessCapsule witness = new WitnessCapsule(address); + witness.setIsJobs(true); + if (w.hasPath(WITNESS_VOTE) && w.getLong(WITNESS_VOTE) > 0) { + witness.setVoteCount(w.getLong(WITNESS_VOTE)); + } + if (w.hasPath(WITNESS_URL)) { + witness.setUrl(w.getString(WITNESS_URL)); + } + witnessStore.put(address.toByteArray(), witness.getData()); + witnessList.add(witness.getAddress()); + }); + + witnessList.sort(Comparator.comparingLong((ByteString b) -> + new WitnessCapsule(witnessStore.get(b.toByteArray())).getVoteCount()) + .reversed() + .thenComparing(Comparator.comparingInt(ByteString::hashCode).reversed())); + List activeWitnesses = witnessList.subList(0, + witnesses.size() >= MAX_ACTIVE_WITNESS_NUM ? MAX_ACTIVE_WITNESS_NUM : witnessList.size()); + witnessScheduleStore.put(ACTIVE_WITNESSES, getActiveWitness(activeWitnesses)); + logger.info("{} witnesses and {} active witnesses have been modified.", + witnesses.size(), activeWitnesses.size()); + spec.commandLine().getOut().format("%d witnesses and %d active witnesses have been modified.", + witnesses.size(), activeWitnesses.size()).println(); + } + + if (forkConfig.hasPath(ACCOUNTS_KEY)) { + List accounts = forkConfig.getConfigList(ACCOUNTS_KEY); + if (accounts.isEmpty()) { + spec.commandLine().getOut().println("no account listed in the config."); + } + + accounts = accounts.stream() + .filter(c -> c.hasPath(ACCOUNT_ADDRESS)) + .collect(Collectors.toList()); + + if (accounts.isEmpty()) { + spec.commandLine().getOut().println("no account listed in the config."); + } + + accounts.stream().forEach( + a -> { + byte[] address = Commons.decodeFromBase58Check(a.getString(ACCOUNT_ADDRESS)); + byte[] value = accountStore.get(address); + Account account = null; + try { + account = ArrayUtils.isEmpty(value) ? null : Account.parseFrom(value); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + System.exit(-1); + } + + if (Objects.isNull(account)) { + ByteString byteAddress = ByteString.copyFrom( + Commons.decodeFromBase58Check(a.getString(ACCOUNT_ADDRESS))); + account = Account.newBuilder().setAddress(byteAddress).build(); + } + AccountCapsule accountCapsule = new AccountCapsule(account); + + if (a.hasPath(ACCOUNT_BALANCE) && a.getLong(ACCOUNT_BALANCE) > 0) { + accountCapsule.setBalance(a.getLong(ACCOUNT_BALANCE)); + } + if (a.hasPath(ACCOUNT_NAME)) { + accountCapsule.setAccountName( + ByteArray.fromString(a.getString(ACCOUNT_NAME))); + } + if (a.hasPath(ACCOUNT_TYPE)) { + accountCapsule.updateAccountType( + AccountType.valueOf(a.getString(ACCOUNT_TYPE))); + } + + if (a.hasPath(ACCOUNT_OWNER)) { + byte[] owner = Commons.decodeFromBase58Check(a.getString(ACCOUNT_OWNER)); + Permission ownerPermission = AccountCapsule + .createDefaultOwnerPermission(ByteString.copyFrom(owner)); + accountCapsule.updatePermissions(ownerPermission, null, null); + } + + if (a.hasPath(ACCOUNT_TRC10_ID) && a.hasPath(ACCOUNT_TRC10_BALANCE) + && a.getLong(ACCOUNT_TRC10_BALANCE) > 0) { + String trc10Id = a.getString(ACCOUNT_TRC10_ID); + if (assetIssueV2Store.get(ByteArray.fromString(trc10Id)) != null) { + if (accountCapsule.getAssetOptimized()) { + byte[] k = Bytes.concat(address, ByteArray.fromString(trc10Id)); + accountAssetStore.put(k, Longs.toByteArray(a.getLong(ACCOUNT_TRC10_BALANCE))); + } else { + Map assetMapV2 = new HashMap<>(account.getAssetV2Map()); + assetMapV2.put(trc10Id, a.getLong(ACCOUNT_TRC10_BALANCE)); + accountCapsule.clearAssetV2(); + accountCapsule.addAssetMapV2(assetMapV2); + } + } else { + logger.info("TRC10: {} not exists in the database.", trc10Id); + spec.commandLine().getOut().format("TRC10: %s not exists in the database.", trc10Id) + .println(); + } + } + + accountStore.put(address, accountCapsule.getData()); + }); + logger.info("{} accounts have been modified.", accounts.size()); + spec.commandLine().getOut().format("%d accounts have been modified.", accounts.size()) + .println(); + } + + if (forkConfig.hasPath(TRC20_CONTRACTS_KEY)) { + List trc20Contracts = forkConfig.getConfigList(TRC20_CONTRACTS_KEY); + if (trc20Contracts.isEmpty()) { + spec.commandLine().getOut().println("no TRC20 contract listed in the config."); + } + + trc20Contracts = trc20Contracts.stream() + .filter(c -> c.hasPath(TRC20_CONTRACT_ADDRESS) && c.hasPath(TRC20_BALANCES_POSITION) && + c.hasPath(TRC20_ACCOUNT) && c.hasPath(TRC20_BALANCE)) + .collect(Collectors.toList()); + + if (trc20Contracts.isEmpty()) { + spec.commandLine().getOut().println("no TRC20 contract listed in the config."); + } + + AtomicInteger cnt = new AtomicInteger(); + + trc20Contracts.stream().forEach( + contract -> { + byte[] contractAddress = Commons + .decodeFromBase58Check(contract.getString(TRC20_CONTRACT_ADDRESS)); + if (contractStore.get(contractAddress) == null) { + spec.commandLine().getErr().format("TRC20 contract: %s not exists in the database.", + contract.getString(TRC20_CONTRACT_ADDRESS)).println(); + return; + } + + SmartContract smartContract; + try { + smartContract = SmartContract.parseFrom(contractStore.get(contractAddress)); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + return; + } + + int balancesSlotPosition = 0; + if (contract.getInt(TRC20_BALANCES_POSITION) > 0) { + balancesSlotPosition = contract.getInt(TRC20_BALANCES_POSITION); + } + byte[] addressWithPrefix = Commons + .decodeFromBase58Check(contract.getString(TRC20_ACCOUNT)); + byte[] address = ByteArray.subArray(addressWithPrefix, 1, 21); + String paddedAddress = String + .format("%064x", new BigInteger(ByteArray.toHexString(address), 16)); + String paddedSlot = String.format("%064x", balancesSlotPosition); + byte[] contractKey = Hash.sha3(ByteArray.fromHexString( + paddedAddress + paddedSlot)); + + byte[] addressHash; + byte[] trxHash = smartContract.getTrxHash().toByteArray(); + if (ByteUtil.isNullOrZeroArray(trxHash)) { + addressHash = Hash.sha3(contractAddress); + } else { + addressHash = Hash.sha3(ByteUtil.merge(contractAddress, trxHash)); + } + + int contractVersion = smartContract.getVersion(); + if (contractVersion == 1) { + contractKey = Hash.sha3(contractKey); + } + byte[] rowKey = new byte[contractKey.length]; + arraycopy(addressHash, 0, rowKey, 0, 16); + arraycopy(contractKey, 16, rowKey, 16, 16); + + String paddedBalance = String + .format("%064x", new BigInteger(contract.getString(TRC20_BALANCE), 10)); + byte[] rowValue = ByteArray.fromHexString(paddedBalance); + StorageRowCapsule storageRowCapsule = new StorageRowCapsule(rowKey, rowValue); + + storageRowStore.put(rowKey, storageRowCapsule.getData()); + cnt.getAndIncrement(); + }); + logger.info("{} TRC20 contracts have been modified.", cnt.get()); + spec.commandLine().getOut() + .format("%d TRC20 contracts have been modified.", cnt.get()) + .println(); + } + + if (forkConfig.hasPath(LATEST_BLOCK_TIMESTAMP) + && forkConfig.getLong(LATEST_BLOCK_TIMESTAMP) > 0) { + long latestBlockHeaderTimestamp = forkConfig.getLong(LATEST_BLOCK_TIMESTAMP); + dynamicPropertiesStore + .put(LATEST_BLOCK_HEADER_TIMESTAMP, ByteArray.fromLong(latestBlockHeaderTimestamp)); + logger.info("The latest block header timestamp has been modified as {}.", + latestBlockHeaderTimestamp); + spec.commandLine().getOut().format("The latest block header timestamp has been modified " + + "as %d.", latestBlockHeaderTimestamp).println(); + } + + if (forkConfig.hasPath(MAINTENANCE_INTERVAL) + && forkConfig.getLong(MAINTENANCE_INTERVAL) > 0) { + long maintenanceTimeInterval = forkConfig.getLong(MAINTENANCE_INTERVAL); + dynamicPropertiesStore + .put(MAINTENANCE_TIME_INTERVAL, ByteArray.fromLong(maintenanceTimeInterval)); + logger.info("The maintenance time interval has been modified as {}.", + maintenanceTimeInterval); + spec.commandLine().getOut().format("The maintenance time interval has been modified as %d.", + maintenanceTimeInterval).println(); + } + + if (forkConfig.hasPath(NEXT_MAINTENANCE_TIME) + && forkConfig.getLong(NEXT_MAINTENANCE_TIME) > 0) { + long nextMaintenanceTime = forkConfig.getLong(NEXT_MAINTENANCE_TIME); + dynamicPropertiesStore.put(MAINTENANCE_TIME, ByteArray.fromLong(nextMaintenanceTime)); + logger.info("The next maintenance time has been modified as {}.", + nextMaintenanceTime); + spec.commandLine().getOut().format("The next maintenance time has been modified as %d.", + nextMaintenanceTime).println(); + } + + DbTool.close(); + return 0; + } + + public static byte[] getActiveWitness(List witnesses) { + byte[] ba = new byte[witnesses.size() * Constant.ADDRESS_BYTE_ARRAY_LENGTH]; + int i = 0; + for (ByteString address : witnesses) { + System.arraycopy(address.toByteArray(), 0, + ba, i * Constant.ADDRESS_BYTE_ARRAY_LENGTH, Constant.ADDRESS_BYTE_ARRAY_LENGTH); + i++; + } + return ba; + } +} diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/DbQuery.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbQuery.java new file mode 100644 index 00000000..16dc2ff3 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbQuery.java @@ -0,0 +1,702 @@ +package org.tron.plugins; + +import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; +import static org.tron.plugins.utils.Constant.ALLOW_OLD_REWARD_OPT; +import static org.tron.plugins.utils.Constant.BLOCK_INDEX_STORE; +import static org.tron.plugins.utils.Constant.BLOCK_PRODUCED_INTERVAL; +import static org.tron.plugins.utils.Constant.BLOCK_STORE; +import static org.tron.plugins.utils.Constant.CHANGE_DELEGATION; +import static org.tron.plugins.utils.Constant.CURRENT_CYCLE_NUMBER; +import static org.tron.plugins.utils.Constant.DECIMAL_OF_VI_REWARD; +import static org.tron.plugins.utils.Constant.DELEGATION_STORE; +import static org.tron.plugins.utils.Constant.DYNAMIC_PROPERTY_STORE; +import static org.tron.plugins.utils.Constant.LATEST_BLOCK_HEADER_NUMBER; +import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME_INTERVAL; +import static org.tron.plugins.utils.Constant.NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE; +import static org.tron.plugins.utils.Constant.REWARDS_KEY; +import static org.tron.plugins.utils.Constant.VOTES_ALL_WITNESSES; +import static org.tron.plugins.utils.Constant.VOTES_STORE; +import static org.tron.plugins.utils.Constant.VOTES_WITNESS_LIST; +import static org.tron.plugins.utils.Constant.WITNESS_STORE; + +import com.google.common.collect.Maps; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.bouncycastle.util.encoders.Hex; +import org.rocksdb.RocksDBException; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.Commons; +import org.tron.common.utils.Pair; +import org.tron.common.utils.Sha256Hash; +import org.tron.common.utils.StringUtil; +import org.tron.core.capsule.AccountCapsule; +import org.tron.core.capsule.BlockCapsule; +import org.tron.core.capsule.VotesCapsule; +import org.tron.core.capsule.WitnessCapsule; +import org.tron.core.exception.BadItemException; +import org.tron.core.store.DelegationStore; +import org.tron.plugins.utils.FileUtils; +import org.tron.plugins.utils.JsonFormat; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DBIterator; +import org.tron.plugins.utils.db.DbTool; +import org.tron.protos.Protocol.Transaction.Contract.ContractType; +import org.tron.protos.contract.WitnessContract.VoteWitnessContract; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +@Slf4j(topic = "query") +@Command(name = "query", + description = "query the latest vote and reward information from the database.", + exitCodeListHeading = "Exit Codes:%n", + exitCodeList = { + "0:Successful", + "n:Internal error: exception occurred, please check logs/toolkit.log"}) +public class DbQuery implements Callable { + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + + @CommandLine.Option(names = {"-d", "--database-directory"}, + defaultValue = "output-directory", + description = "java-tron database directory path. Default: ${DEFAULT-VALUE}") + private String database; + + @CommandLine.Option(names = {"-c", "--config"}, + defaultValue = "query.conf", + description = "config the votes and reward query options." + + " Default: ${DEFAULT-VALUE}") + private String config; + + @CommandLine.Option(names = {"-h", "--help"}) + private boolean help; + + private DBInterface witnessStore; + private DBInterface votesStore; + private DBInterface dynamicPropertiesStore; + private DBInterface blockIndexStore; + private DBInterface blockStore; + private DBInterface accountStore; + private DBInterface delegationStore; + + boolean allWitness = false; + List witnessList = new ArrayList<>(); + private Map voters = new HashMap<>(); + private Map votesTx = new HashMap<>(); + + List rewardAddressList = new ArrayList<>(); + Map latestWitnessVi = new HashMap<>(); + + private void initStore() throws IOException, RocksDBException { + String srcDir = database + File.separator + "database"; + witnessStore = DbTool.getDB(srcDir, WITNESS_STORE); + votesStore = DbTool.getDB(srcDir, VOTES_STORE); + dynamicPropertiesStore = DbTool.getDB(srcDir, DYNAMIC_PROPERTY_STORE); + blockIndexStore = DbTool.getDB(srcDir, BLOCK_INDEX_STORE); + blockStore = DbTool.getDB(srcDir, BLOCK_STORE); + accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); + delegationStore = DbTool.getDB(srcDir, DELEGATION_STORE); + } + + + @Override + public Integer call() throws Exception { + if (help) { + spec.commandLine().usage(System.out); + return 0; + } + + Config queryConfig; + File file = Paths.get(config).toFile(); + if (file.exists() && file.isFile()) { + queryConfig = ConfigFactory.parseFile(file); + } else { + logger.error("Query config file [" + config + "] not exists!"); + spec.commandLine().getErr().format("Fork config file: %s not exists!", config).println(); + return 1; + } + + File dbFile = Paths.get(database).toFile(); + if (!dbFile.exists() || !dbFile.isDirectory()) { + logger.error("Database [" + database + "] not exists!"); + spec.commandLine().getErr().format("Database %s not exists!", database).println(); + return 1; + } + File tmp = Paths.get(database, "database", "tmp").toFile(); + if (tmp.exists()) { + FileUtils.deleteDir(tmp); + } + + initStore(); + processVotes(queryConfig); + processRewards(queryConfig); + + DbTool.close(); + return 0; + } + + private void processVotes(Config queryConfig) throws BadItemException { + if (queryConfig.hasPath(VOTES_ALL_WITNESSES)) { + allWitness = queryConfig.getBoolean(VOTES_ALL_WITNESSES); + } + if (queryConfig.hasPath(VOTES_WITNESS_LIST)) { + witnessList = queryConfig.getStringList(VOTES_WITNESS_LIST); + } + + if (!allWitness && witnessList.isEmpty()) { + spec.commandLine().getOut() + .println("skip the vote query."); + logger.info("skip the vote query."); + return; + } + + Map witnesses = new HashMap<>(); + Map oldWitnessCnt = new HashMap<>(); + DBIterator iterator = witnessStore.iterator(); + WitnessCapsule witnessCapsule; + for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { + witnessCapsule = new WitnessCapsule(iterator.getValue()); + witnesses.put(ByteString.copyFrom(iterator.getKey()), witnessCapsule); + oldWitnessCnt.put(witnessCapsule.getAddress(), witnessCapsule.getVoteCount()); + } + + Map countWitness = countVote(); + loadVotesTx(); + + AtomicInteger cnt = new AtomicInteger(); + votesTx.forEach((address, voteWitnessTx) -> { + if (allWitness || existInWitnessList(voters.get(address), witnessList)) { + cnt.getAndIncrement(); + } + }); + spec.commandLine().getOut() + .format("There are %d related system-contract vote txs", cnt.get()) + .println(); + logger.info("There are {} related system-contract vote txs", cnt.get()); + + cnt.set(-1); + votesTx.forEach((address, voteWitnessTx) -> { + VotesCapsule votesCapsule = voters.get(address); + if (!allWitness && !existInWitnessList(votesCapsule, witnessList)) { + return; + } + cnt.getAndIncrement(); + String txHashStr = voteWitnessTx.txHash.toString(); + spec.commandLine().getOut().format("tx %d: %s", cnt.get(), txHashStr).println(); + logger.info("tx {}: {}", cnt.get(), txHashStr); +// String voteWitnessStr = JsonFormat.printToString(voteWitnessTx.voteWitnessContract, true); +// spec.commandLine().getOut().println(voteWitnessStr); +// logger.info(voteWitnessStr); + String voteCapsuleStr = JsonFormat.printToString(votesCapsule.getInstance(), true); + spec.commandLine().getOut().println(voteCapsuleStr); + logger.info(voteCapsuleStr); + voters.remove(address); + }); + + cnt.set(0); + voters.forEach((address, votesCapsule) -> { + if ((allWitness || existInWitnessList(voters.get(address), witnessList)) + && votesTx.get(address) == null) { + cnt.getAndIncrement(); + } + }); + + spec.commandLine().getOut() + .format("There are %d related smart-contract votes txs", cnt.get()) + .println(); + logger.info("There are {} related smart-contract votes txs", cnt.get()); + cnt.set(-1); + voters.forEach((address, voteCapsule) -> { + if (!allWitness && !existInWitnessList(voteCapsule, witnessList)) { + return; + } + cnt.getAndIncrement(); + spec.commandLine().getOut().format("tx %d", cnt.get()).println(); + logger.info("tx {}:", cnt.get()); + String voteCapsuleStr = JsonFormat.printToString(voteCapsule.getInstance(), true); + spec.commandLine().getOut().println(voteCapsuleStr); + logger.info(voteCapsuleStr); + }); + + countWitness.forEach((address, voteCount) -> { + WitnessCapsule witness = witnesses.get(address); + if (witness == null) { + return; + } + witness.setVoteCount(witness.getVoteCount() + voteCount); + witnesses.put(address, witness); + }); + + spec.commandLine().getOut().println("Display the witness list with latest vote count: "); + logger.info("Display the witness list with latest vote count: "); + List witnessCapsuleList; + if (allWitness) { + witnessCapsuleList = new ArrayList<>(witnesses.values()); + Collections.sort(witnessCapsuleList, + Comparator.comparingLong(WitnessCapsule::getVoteCount).reversed()); + } else { + List finalWitnessCapsuleList = new ArrayList<>(); + witnessList.forEach( + witness -> { + ByteString address = ByteString.copyFrom(Commons.decodeFromBase58Check(witness)); + WitnessCapsule witnessTmp = witnesses.get(address); + if (witnessTmp == null) { + spec.commandLine().getErr().format("address: %s is not witness", witness).println(); + logger.error("address: {} is not witness", witness); + return; + } + finalWitnessCapsuleList.add(witnessTmp); + } + ); + witnessCapsuleList = finalWitnessCapsuleList; + } + +// WitnessList.Builder builder = WitnessList.newBuilder(); +// witnessCapsuleList.forEach(witness -> builder.addWitnesses(witness.getInstance())); +// String witnessesJson = JsonFormat.printToString(builder.build(), true); +// spec.commandLine().getOut().println(witnessesJson); +// logger.info(witnessesJson); + cnt.set(-1); + witnessCapsuleList.forEach( + witness -> { + cnt.getAndIncrement(); + String output = formatWitness(witness, oldWitnessCnt.get(witness.getAddress())); + spec.commandLine().getOut().println(cnt.get() + " " + output); + logger.info(cnt.get() + " " + output); + }); + } + + private String formatWitness(WitnessCapsule witnessCapsule, long previousCnt) { + return String.format("{\"address\": %s," + + "\"oldVoteCount\": %d,\"newVoteCount\": %d," + + "\"voteCountIncrement\": %d,\"url\": %s," + + "\"totalProduced\": %d,\"totalMissed\": %d,\"latestBlockNum\": %d," + + "\"latestSlotNum\": %d,\"isJobs\": %b}", + StringUtil.encode58Check(witnessCapsule.getAddress().toByteArray()), + previousCnt, + witnessCapsule.getVoteCount(), + witnessCapsule.getVoteCount() - previousCnt, + witnessCapsule.getUrl(), + witnessCapsule.getTotalProduced(), + witnessCapsule.getTotalMissed(), + witnessCapsule.getLatestBlockNum(), + witnessCapsule.getLatestSlotNum(), + witnessCapsule.getIsJobs()); + } + + + private Map countVote() { + final Map countWitness = Maps.newHashMap(); + DBIterator dbIterator = votesStore.iterator(); + long sizeCount = 0; + dbIterator.seekToFirst(); + while (dbIterator.hasNext()) { + Entry next = dbIterator.next(); + VotesCapsule votes = new VotesCapsule(next.getValue()); + voters.put(ByteString.copyFrom(next.getKey()), votes); + + votes.getOldVotes().forEach(vote -> { + ByteString voteAddress = vote.getVoteAddress(); + long voteCount = vote.getVoteCount(); + if (countWitness.containsKey(voteAddress)) { + countWitness.put(voteAddress, countWitness.get(voteAddress) - voteCount); + } else { + countWitness.put(voteAddress, -voteCount); + } + }); + votes.getNewVotes().forEach(vote -> { + ByteString voteAddress = vote.getVoteAddress(); + long voteCount = vote.getVoteCount(); + if (countWitness.containsKey(voteAddress)) { + countWitness.put(voteAddress, countWitness.get(voteAddress) + voteCount); + } else { + countWitness.put(voteAddress, voteCount); + } + }); + sizeCount++; + } + spec.commandLine().getOut().format("There are total %d new votes in this epoch", sizeCount) + .println(); + logger.info("There are total {} new votes in this epoch", sizeCount); + return countWitness; + } + + private void loadVotesTx() throws BadItemException { + long maintenanceTimeInterval = ByteArray + .toLong(dynamicPropertiesStore.get(MAINTENANCE_TIME_INTERVAL)); + long maintenanceBlockCnt = maintenanceTimeInterval / BLOCK_PRODUCED_INTERVAL; + long latestBlockNumber = ByteArray + .toLong(dynamicPropertiesStore.get(LATEST_BLOCK_HEADER_NUMBER)); + long startBlock = + latestBlockNumber > maintenanceBlockCnt ? latestBlockNumber - maintenanceBlockCnt : 0; + long block; + for (block = startBlock; block <= latestBlockNumber; block++) { + byte[] blockHash = blockIndexStore.get(ByteArray.fromLong(block)); + BlockCapsule blockCapsule = new BlockCapsule(blockStore.get(blockHash)); + + blockCapsule.getTransactions().forEach(txCapsule -> { + ContractType txType = txCapsule.getInstance().getRawData().getContract(0).getType(); + if (!txType.equals(ContractType.VoteWitnessContract)) { + return; + } + try { + VoteWitnessContract voteWitnessContract = txCapsule.getInstance().getRawData() + .getContract(0) + .getParameter().unpack(VoteWitnessContract.class); + ByteString ownerAddress = voteWitnessContract.getOwnerAddress(); + voteWitnessContract.getVotesList().forEach(vote -> { + if (voters.keySet().contains(ownerAddress)) { + votesTx.put(ownerAddress, + new VoteWitnessTx(txCapsule.getTransactionId(), voteWitnessContract)); + } + }); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + System.exit(-1); + } + }); + } + } + + private class VoteWitnessTx { + + private Sha256Hash txHash; + private VoteWitnessContract voteWitnessContract; + + VoteWitnessTx(Sha256Hash txHash, VoteWitnessContract voteWitnessContract) { + this.txHash = txHash; + this.voteWitnessContract = voteWitnessContract; + } + + private Boolean existInWitnessList(List witnessList) { + AtomicBoolean exist = new AtomicBoolean(false); + voteWitnessContract.getVotesList().forEach(vote -> { + if (witnessList.contains(StringUtil.encode58Check(vote.getVoteAddress().toByteArray()))) { + exist.set(true); + } + }); + return exist.get(); + } + } + + private Boolean existInWitnessList(VotesCapsule votesCapsule, List witnessList) { + AtomicBoolean exist = new AtomicBoolean(false); + votesCapsule.getOldVotes().forEach(vote -> { + if (witnessList.contains(StringUtil.encode58Check(vote.getVoteAddress().toByteArray()))) { + exist.set(true); + } + }); + votesCapsule.getNewVotes().forEach(vote -> { + if (witnessList.contains(StringUtil.encode58Check(vote.getVoteAddress().toByteArray()))) { + exist.set(true); + } + }); + return exist.get(); + } + + private void processRewards(Config config) { + if (config.hasPath(REWARDS_KEY)) { + rewardAddressList = config.getStringList(REWARDS_KEY); + } + if (rewardAddressList.isEmpty()) { + spec.commandLine().getOut() + .println("skip the reward query."); + logger.info("skip the reward query."); + return; + } + spec.commandLine().getOut().println("\nBegin to query the reward"); + + loadAccumulateWitnessVi(); + rewardAddressList.forEach(address -> { + long reward = queryReward(Commons.decodeFromBase58Check(address), false); + long latestReward = queryReward(Commons.decodeFromBase58Check(address), true); + spec.commandLine().getOut() + .format("address: %s, cycle reward: %d, latest reward: %d, increment: %d", address, + reward, latestReward, (latestReward - reward)).println(); + logger.info("address: {}, cycle reward: {}, latest reward: {}, increment: {}", address, + reward, latestReward, (latestReward - reward)); + }); + + spec.commandLine().getOut().println("Finish querying the reward"); + } + + private void loadAccumulateWitnessVi() { + long cycle = getCurrentCycle(); + DBIterator iterator = witnessStore.iterator(); + WitnessCapsule witnessCapsule; + for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { + witnessCapsule = new WitnessCapsule(iterator.getValue()); + accumulateWitnessVi(cycle, witnessCapsule.createDbKey(), witnessCapsule.getVoteCount()); + } + } + + private void accumulateWitnessVi(long cycle, byte[] address, long voteCount) { + BigInteger preVi = getWitnessVi(cycle - 1, address); + long reward = getReward(cycle, address); + if (reward == 0 || voteCount == 0) { // Just forward pre vi + if (!BigInteger.ZERO.equals(preVi)) { // Zero vi will not be record + setWitnessVi(cycle, address, preVi); + } + } else { // Accumulate delta vi + BigInteger deltaVi = BigInteger.valueOf(reward) + .multiply(DECIMAL_OF_VI_REWARD) + .divide(BigInteger.valueOf(voteCount)); + setWitnessVi(cycle, address, preVi.add(deltaVi)); + } + } + + private void setWitnessVi(long cycle, byte[] address, BigInteger value) { + byte[] key = buildViKey(cycle, address); + latestWitnessVi.put(ByteArray.toHexString(key), value); + } + + private BigInteger getWitnessViFromMap(long cycle, byte[] address) { + byte[] key = buildViKey(cycle, address); + BigInteger value = latestWitnessVi.get(ByteArray.toHexString(key)); + if (value == null) { + return BigInteger.ZERO; + } else { + return value; + } + } + + private long queryReward(byte[] address, boolean isLatest) { + if (ByteArray.toLong(dynamicPropertiesStore.get(CHANGE_DELEGATION)) != 1) { + return 0; + } + + byte[] accountValue = accountStore.get(address); + if (accountValue == null) { + return 0; + } + AccountCapsule accountCapsule = new AccountCapsule(accountValue); + long beginCycle = getBeginCycle(address); + long endCycle = getEndCycle(address); + long currentCycle = getCurrentCycle(); + + long reward = 0; + if (beginCycle > currentCycle) { + return accountCapsule.getAllowance(); + } + + //withdraw the latest cycle reward + if (beginCycle + 1 == endCycle && beginCycle < currentCycle) { + AccountCapsule account = getAccountVote(beginCycle, address); + if (account != null) { + reward = computeReward(beginCycle, endCycle, account, false); + } + beginCycle += 1; + } + + endCycle = currentCycle; + if (CollectionUtils.isEmpty(accountCapsule.getVotesList())) { + return reward + accountCapsule.getAllowance(); + } + if ((!isLatest && beginCycle < endCycle) || (isLatest && beginCycle <= endCycle)) { + reward += computeReward(beginCycle, endCycle, accountCapsule, isLatest); + } + return reward + accountCapsule.getAllowance(); + } + + private long getBeginCycle(byte[] address) { + byte[] beginCycleValue = delegationStore.get(address); + return beginCycleValue == null ? 0 : ByteArray.toLong(beginCycleValue); + } + + private long getEndCycle(byte[] address) { + byte[] endCycleValue = delegationStore.get(buildEndCycleKey(address)); + return endCycleValue == null ? -1L : ByteArray.toLong(endCycleValue); + } + + private byte[] buildEndCycleKey(byte[] address) { + return ("end-" + Hex.toHexString(address)).getBytes(); + } + + private long getCurrentCycle() { + byte[] currentCycleValue = dynamicPropertiesStore.get(CURRENT_CYCLE_NUMBER); + return currentCycleValue == null ? 0L : ByteArray.toLong(currentCycleValue); + } + + private AccountCapsule getAccountVote(long cycle, byte[] address) { + byte[] value = delegationStore.get(buildAccountVoteKey(cycle, address)); + if (value == null) { + return null; + } else { + return new AccountCapsule(value); + } + } + + private byte[] buildAccountVoteKey(long cycle, byte[] address) { + return (cycle + "-" + Hex.toHexString(address) + "-account-vote").getBytes(); + } + + private long getNewRewardAlgorithmEffectiveCycle() { + byte[] value = dynamicPropertiesStore.get(NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE); + return value == null ? 0L : ByteArray.toLong(value); + } + + private Boolean allowOldRewardOpt() { + byte[] value = dynamicPropertiesStore.get(ALLOW_OLD_REWARD_OPT); + return value != null && ByteArray.toLong(value) == 1; + } + + private BigInteger getWitnessVi(long cycle, byte[] address) { + byte[] value = delegationStore.get(buildViKey(cycle, address)); + if (value == null) { + return BigInteger.ZERO; + } else { + return new BigInteger(value); + } + } + + private byte[] buildViKey(long cycle, byte[] address) { + return (cycle + "-" + Hex.toHexString(address) + "-vi").getBytes(); + } + + public long getReward(long cycle, byte[] address) { + byte[] value = delegationStore.get(buildRewardKey(cycle, address)); + if (value == null) { + return 0L; + } else { + return ByteArray.toLong(value); + } + } + + private byte[] buildRewardKey(long cycle, byte[] address) { + return (cycle + "-" + Hex.toHexString(address) + "-reward").getBytes(); + } + + public long getWitnessVote(long cycle, byte[] address) { + byte[] value = delegationStore.get(buildVoteKey(cycle, address)); + if (value == null) { + return -1L; + } else { + return ByteArray.toLong(value); + } + } + + private byte[] buildVoteKey(long cycle, byte[] address) { + return (cycle + "-" + Hex.toHexString(address) + "-vote").getBytes(); + } + + + /** + * Compute reward from begin cycle to end cycle, which endCycle must greater than beginCycle. + * While computing reward after new reward algorithm taking effective cycle number, it will use + * new algorithm instead of old way. + * + * @param beginCycle begin cycle (include) + * @param endCycle end cycle (exclude) + * @param accountCapsule account capsule + * @param isLatest whether to compute the latest reward + * @return total reward + */ + private long computeReward(long beginCycle, long endCycle, AccountCapsule accountCapsule, + boolean isLatest) { + if ((!isLatest && beginCycle >= endCycle) || (isLatest && beginCycle > endCycle)) { + return 0; + } + + long reward = 0; + long newAlgorithmCycle = getNewRewardAlgorithmEffectiveCycle(); + List> srAddresses = accountCapsule.getVotesList().stream() + .map(vote -> new Pair<>(vote.getVoteAddress().toByteArray(), vote.getVoteCount())) + .collect(Collectors.toList()); + if (beginCycle < newAlgorithmCycle) { + long oldEndCycle = Math.min(endCycle, newAlgorithmCycle); + reward = getOldReward(beginCycle, oldEndCycle, srAddresses); + beginCycle = oldEndCycle; + } + if ((!isLatest && beginCycle < endCycle) || (isLatest && beginCycle <= endCycle)) { + for (Pair vote : srAddresses) { + byte[] srAddress = vote.getKey(); + BigInteger beginVi = getWitnessVi(beginCycle - 1, srAddress); + BigInteger endVi; + if (!isLatest) { + endVi = getWitnessVi(endCycle - 1, srAddress); + } else { + endCycle = getCurrentCycle(); + endVi = getWitnessViFromMap(endCycle, srAddress); + } + BigInteger deltaVi = endVi.subtract(beginVi); + if (deltaVi.signum() <= 0) { + continue; + } + long userVote = vote.getValue(); + reward += deltaVi.multiply(BigInteger.valueOf(userVote)) + .divide(DECIMAL_OF_VI_REWARD).longValue(); + } + } + return reward; + } + + private long getNewRewardAlgorithmReward(long beginCycle, long endCycle, + List> votes) { + long reward = 0; + if (beginCycle < endCycle) { + for (Pair vote : votes) { + byte[] srAddress = vote.getKey(); + BigInteger beginVi = getWitnessVi(beginCycle - 1, srAddress); + BigInteger endVi = getWitnessVi(endCycle - 1, srAddress); + BigInteger deltaVi = endVi.subtract(beginVi); + if (deltaVi.signum() <= 0) { + continue; + } + long userVote = vote.getValue(); + reward += deltaVi.multiply(BigInteger.valueOf(userVote)) + .divide(DECIMAL_OF_VI_REWARD).longValue(); + } + } + return reward; + } + + private long computeReward(long cycle, List> votes) { + long reward = 0; + for (Pair vote : votes) { + byte[] srAddress = vote.getKey(); + long totalReward = getReward(cycle, srAddress); + if (totalReward <= 0) { + continue; + } + long totalVote = getWitnessVote(cycle, srAddress); + if (totalVote == DelegationStore.REMARK || totalVote == 0) { + continue; + } + long userVote = vote.getValue(); + // Replace floating-point division with integer-based calculation + reward += (userVote * totalReward) / totalVote; + } + return reward; + } + + private long getOldReward(long begin, long end, List> votes) { + if (allowOldRewardOpt()) { + return getNewRewardAlgorithmReward(begin, end, votes); + } + long reward = 0; + for (long cycle = begin; cycle < end; cycle++) { + reward += computeReward(cycle, votes); + } + return reward; + } + +} diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/Constant.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/Constant.java new file mode 100644 index 00000000..7fcc380c --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/Constant.java @@ -0,0 +1,63 @@ +package org.tron.plugins.utils; + +import java.math.BigInteger; + +public class Constant { + + public static final String WITNESS_KEY = "witnesses"; + public static final String WITNESS_ADDRESS = "address"; + public static final String WITNESS_URL = "url"; + public static final String WITNESS_VOTE = "voteCount"; + public static final String ACCOUNTS_KEY = "accounts"; + public static final String ACCOUNT_NAME = "accountName"; + public static final String ACCOUNT_TYPE = "accountType"; + public static final String ACCOUNT_ADDRESS = "address"; + public static final String ACCOUNT_BALANCE = "balance"; + public static final String ACCOUNT_OWNER = "owner"; + public static final String ACCOUNT_TRC10_ID = "trc10Id"; + public static final String ACCOUNT_TRC10_BALANCE = "trc10Balance"; + + public static final String TRC20_CONTRACTS_KEY = "trc20Contracts"; + public static final String TRC20_CONTRACT_ADDRESS = "contractAddress"; + public static final String TRC20_BALANCES_POSITION = "balancesSlotPosition"; + public static final String TRC20_ACCOUNT = "address"; + public static final String TRC20_BALANCE = "balance"; + + public static final String LATEST_BLOCK_TIMESTAMP = "latestBlockHeaderTimestamp"; + public static final String MAINTENANCE_INTERVAL = "maintenanceTimeInterval"; + public static final String NEXT_MAINTENANCE_TIME = "nextMaintenanceTime"; + public static final int MAX_ACTIVE_WITNESS_NUM = 27; + + public static final String WITNESS_STORE = "witness"; + public static final String WITNESS_SCHEDULE_STORE = "witness_schedule"; + public static final String ACCOUNT_STORE = "account"; + public static final String DYNAMIC_PROPERTY_STORE = "properties"; + public static final String ASSET_ISSUE_V2 = "asset-issue-v2"; + public static final String ACCOUNT_ASSET = "account-asset"; + public static final String CONTRACT_STORE = "contract"; + public static final String STORAGE_ROW_STORE = "storage-row"; + public static final String VOTES_STORE = "votes"; + public static final String BLOCK_INDEX_STORE = "block-index"; + public static final String BLOCK_STORE = "block"; + public static final String DELEGATION_STORE = "delegation"; + + public static final byte[] LATEST_BLOCK_HEADER_TIMESTAMP = "latest_block_header_timestamp" + .getBytes(); + public static final byte[] MAINTENANCE_TIME_INTERVAL = "MAINTENANCE_TIME_INTERVAL".getBytes(); + public static final byte[] LATEST_BLOCK_HEADER_NUMBER = "latest_block_header_number".getBytes(); + public static final byte[] MAINTENANCE_TIME = "NEXT_MAINTENANCE_TIME".getBytes(); + public static final byte[] ACTIVE_WITNESSES = "active_witnesses".getBytes(); + public static final int ADDRESS_BYTE_ARRAY_LENGTH = 21; + + public static final String VOTES_ALL_WITNESSES = "vote.allWitnesses"; + public static final String VOTES_WITNESS_LIST = "vote.witnessList"; + public static final int BLOCK_PRODUCED_INTERVAL = 3000; + + public static final String REWARDS_KEY = "reward"; + public static final byte[] CHANGE_DELEGATION = "CHANGE_DELEGATION".getBytes(); + public static final byte[] CURRENT_CYCLE_NUMBER = "CURRENT_CYCLE_NUMBER".getBytes(); + public static final byte[] NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE = + "NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE".getBytes(); + public static final byte[] ALLOW_OLD_REWARD_OPT = "ALLOW_OLD_REWARD_OPT".getBytes(); + public static final BigInteger DECIMAL_OF_VI_REWARD = BigInteger.valueOf(10).pow(18); +} diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/HttpSelfFormatFieldName.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/HttpSelfFormatFieldName.java new file mode 100644 index 00000000..c40f10da --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/HttpSelfFormatFieldName.java @@ -0,0 +1,283 @@ +package org.tron.plugins.utils; + +import java.util.HashMap; +import java.util.Map; + +public class HttpSelfFormatFieldName { + + private static Map AddressFieldNameMap = new HashMap<>(); + private static Map NameFieldNameMap = new HashMap<>(); + + static { + //***** api.proto ***** + //DelegatedResourceMessage + AddressFieldNameMap.put("protocol.DelegatedResourceMessage.fromAddress", 1); + AddressFieldNameMap.put("protocol.DelegatedResourceMessage.toAddress", 1); + //TransactionSignWeight + AddressFieldNameMap.put("protocol.TransactionSignWeight.approved_list", 1); + //TransactionApprovedList + AddressFieldNameMap.put("protocol.TransactionApprovedList.approved_list", 1); + //PrivateParameters + AddressFieldNameMap.put("protocol.PrivateParameters.transparent_from_address", 1); + AddressFieldNameMap.put("protocol.PrivateParameters.transparent_to_address", 1); + //PrivateParametersWithoutAsk + AddressFieldNameMap.put("protocol.PrivateParametersWithoutAsk.transparent_from_address", 1); + AddressFieldNameMap.put("protocol.PrivateParametersWithoutAsk.transparent_to_address", 1); + //PrivateShieldedTRC20Parameters + AddressFieldNameMap.put( + "protocol.PrivateShieldedTRC20Parameters.transparent_to_address", 1); + AddressFieldNameMap.put( + "protocol.PrivateShieldedTRC20Parameters.shielded_TRC20_contract_address", 1); + //PrivateShieldedTRC20ParametersWithoutAsk + AddressFieldNameMap.put( + "protocol.PrivateShieldedTRC20ParametersWithoutAsk.transparent_to_address", 1); + AddressFieldNameMap.put( + "protocol.PrivateShieldedTRC20ParametersWithoutAsk.shielded_TRC20_contract_address", 1); + //IvkDecryptTRC20Parameters + AddressFieldNameMap.put( + "protocol.IvkDecryptTRC20Parameters.shielded_TRC20_contract_address", 1); + //OvkDecryptTRC20Parameters + AddressFieldNameMap.put( + "protocol.OvkDecryptTRC20Parameters.shielded_TRC20_contract_address", 1); + //NfTRC20Parameters + AddressFieldNameMap.put( + "protocol.NfTRC20Parameters.shielded_TRC20_contract_address", 1); + //ShieldedTRC20TriggerContractParameters + AddressFieldNameMap.put( + "protocol.ShieldedTRC20TriggerContractParameters.transparent_to_address", 1); + AddressFieldNameMap.put( + "protocol.DecryptNotesTRC20.NoteTx.transparent_to_address", 1); + + //***** Contract.proto ***** + //AccountCreateContract + AddressFieldNameMap.put("protocol.AccountCreateContract.owner_address", 1); + AddressFieldNameMap.put("protocol.AccountCreateContract.account_address", 1); + //AccountUpdateContract + AddressFieldNameMap.put("protocol.AccountUpdateContract.owner_address", 1); + //SetAccountIdContract + AddressFieldNameMap.put("protocol.SetAccountIdContract.owner_address", 1); + //TransferContract + AddressFieldNameMap.put("protocol.TransferContract.owner_address", 1); + AddressFieldNameMap.put("protocol.TransferContract.to_address", 1); + //CancelDeferredTransactionContract + AddressFieldNameMap.put("protocol.CancelDeferredTransactionContract.ownerAddress", 1); + //TransferAssetContract + AddressFieldNameMap.put("protocol.TransferAssetContract.owner_address", 1); + AddressFieldNameMap.put("protocol.TransferAssetContract.to_address", 1); + //VoteAssetContract + AddressFieldNameMap.put("protocol.VoteAssetContract.owner_address", 1); + AddressFieldNameMap.put("protocol.VoteAssetContract.vote_address", 1); + //VoteWitnessContract + AddressFieldNameMap.put("protocol.VoteWitnessContract.Vote.vote_address", 1); + AddressFieldNameMap.put("protocol.VoteWitnessContract.owner_address", 1); + //UpdateSettingContract + AddressFieldNameMap.put("protocol.UpdateSettingContract.owner_address", 1); + AddressFieldNameMap.put("protocol.UpdateSettingContract.contract_address", 1); + //UpdateEnergyLimitContract + AddressFieldNameMap.put("protocol.UpdateEnergyLimitContract.owner_address", 1); + AddressFieldNameMap.put("protocol.UpdateEnergyLimitContract.contract_address", 1); + //ClearABIContract + AddressFieldNameMap.put("protocol.ClearABIContract.owner_address", 1); + AddressFieldNameMap.put("protocol.ClearABIContract.contract_address", 1); + //WitnessCreateContract + AddressFieldNameMap.put("protocol.WitnessCreateContract.owner_address", 1); + //WitnessUpdateContract + AddressFieldNameMap.put("protocol.WitnessUpdateContract.owner_address", 1); + //AssetIssueContract + AddressFieldNameMap.put("protocol.AssetIssueContract.owner_address", 1); + //ParticipateAssetIssueContract + AddressFieldNameMap.put("protocol.ParticipateAssetIssueContract.owner_address", 1); + AddressFieldNameMap.put("protocol.ParticipateAssetIssueContract.to_address", 1); + //FreezeBalanceContract + AddressFieldNameMap.put("protocol.FreezeBalanceContract.owner_address", 1); + AddressFieldNameMap.put("protocol.FreezeBalanceContract.receiver_address", 1); + //UnfreezeBalanceContract + AddressFieldNameMap.put("protocol.UnfreezeBalanceContract.owner_address", 1); + AddressFieldNameMap.put("protocol.UnfreezeBalanceContract.receiver_address", 1); + //UnfreezeAssetContract + AddressFieldNameMap.put("protocol.UnfreezeAssetContract.owner_address", 1); + //WithdrawBalanceContract + AddressFieldNameMap.put("protocol.WithdrawBalanceContract.owner_address", 1); + //UpdateAssetContract + AddressFieldNameMap.put("protocol.UpdateAssetContract.owner_address", 1); + //ProposalCreateContract + AddressFieldNameMap.put("protocol.ProposalCreateContract.owner_address", 1); + //ProposalApproveContract + AddressFieldNameMap.put("protocol.ProposalApproveContract.owner_address", 1); + //ProposalDeleteContract + AddressFieldNameMap.put("protocol.ProposalDeleteContract.owner_address", 1); + //CreateSmartContract + AddressFieldNameMap.put("protocol.CreateSmartContract.owner_address", 1); + //TriggerSmartContract + AddressFieldNameMap.put("protocol.TriggerSmartContract.owner_address", 1); + AddressFieldNameMap.put("protocol.TriggerSmartContract.contract_address", 1); + //BuyStorageContract + AddressFieldNameMap.put("protocol.BuyStorageContract.owner_address", 1); + //BuyStorageBytesContract + AddressFieldNameMap.put("protocol.BuyStorageBytesContract.owner_address", 1); + //SellStorageContract + AddressFieldNameMap.put("protocol.SellStorageContract.owner_address", 1); + //ExchangeCreateContract + AddressFieldNameMap.put("protocol.ExchangeCreateContract.owner_address", 1); + //ExchangeInjectContract + AddressFieldNameMap.put("protocol.ExchangeInjectContract.owner_address", 1); + //ExchangeWithdrawContract + AddressFieldNameMap.put("protocol.ExchangeWithdrawContract.owner_address", 1); + //ExchangeTransactionContract + AddressFieldNameMap.put("protocol.ExchangeTransactionContract.owner_address", 1); + //AccountPermissionUpdateContract + AddressFieldNameMap.put("protocol.AccountPermissionUpdateContract.owner_address", 1); + //UpdateBrokerageContract + AddressFieldNameMap.put("protocol.UpdateBrokerageContract.owner_address", 1); + //ShieldedTransferContract + AddressFieldNameMap.put("protocol.ShieldedTransferContract.transparent_from_address", 1); + AddressFieldNameMap.put("protocol.ShieldedTransferContract.transparent_to_address", 1); + //FreezeBalanceV2Contract + AddressFieldNameMap.put("protocol.FreezeBalanceV2Contract.owner_address", 1); + //UnfreezeBalanceV2Contract + AddressFieldNameMap.put("protocol.UnfreezeBalanceV2Contract.owner_address", 1); + //WithdrawExpireUnfreezeContract + AddressFieldNameMap.put("protocol.WithdrawExpireUnfreezeContract.owner_address", 1); + //DelegateResourceContract + AddressFieldNameMap.put("protocol.DelegateResourceContract.owner_address", 1); + AddressFieldNameMap.put("protocol.DelegateResourceContract.receiver_address", 1); + //UnDelegateResourceContract + AddressFieldNameMap.put("protocol.UnDelegateResourceContract.owner_address", 1); + AddressFieldNameMap.put("protocol.UnDelegateResourceContract.receiver_address", 1); + //CancelAllUnfreezeV2Contract + AddressFieldNameMap.put("protocol.CancelAllUnfreezeV2Contract.owner_address", 1); + AddressFieldNameMap.put("protocol.CanDelegatedMaxSizeRequestMessage.owner_address", 1); + AddressFieldNameMap.put("protocol.GetAvailableUnfreezeCountRequestMessage.owner_address", 1); + AddressFieldNameMap.put("protocol.CanWithdrawUnfreezeAmountRequestMessage.owner_address", 1); + + //***** Tron.proto ***** + //AccountId + AddressFieldNameMap.put("protocol.AccountId.address", 1); + //Vote + AddressFieldNameMap.put("protocol.Vote.vote_address", 1); + //Proposal + AddressFieldNameMap.put("protocol.Proposal.proposer_address", 1); + AddressFieldNameMap.put("protocol.Proposal.approvals", 1); + //Exchange + AddressFieldNameMap.put("protocol.Exchange.creator_address", 1); + //Account + AddressFieldNameMap.put("protocol.Account.address", 1); + //Key + AddressFieldNameMap.put("protocol.Key.address", 1); + //DelegatedResource + AddressFieldNameMap.put("protocol.DelegatedResource.from", 1); + AddressFieldNameMap.put("protocol.DelegatedResource.to", 1); + //Witness + AddressFieldNameMap.put("protocol.Witness.address", 1); + //Votes + AddressFieldNameMap.put("protocol.Votes.address", 1); + //TransactionInfo + AddressFieldNameMap.put("protocol.TransactionInfo.Log.address", 1); + AddressFieldNameMap.put("protocol.TransactionInfo.contract_address", 1); + //DeferredTransaction + AddressFieldNameMap.put("protocol.DeferredTransaction.senderAddress", 1); + AddressFieldNameMap.put("protocol.DeferredTransaction.receiverAddress", 1); + //BlockHeader + AddressFieldNameMap.put("protocol.BlockHeader.raw.witness_address", 1); + //SmartContract + AddressFieldNameMap.put("protocol.SmartContract.origin_address", 1); + AddressFieldNameMap.put("protocol.SmartContract.contract_address", 1); + //InternalTransaction + AddressFieldNameMap.put("protocol.InternalTransaction.caller_address", 1); + AddressFieldNameMap.put("protocol.InternalTransaction.transferTo_address", 1); + //DelegatedResourceAccountIndex + AddressFieldNameMap.put("protocol.DelegatedResourceAccountIndex.account", 1); + AddressFieldNameMap.put("protocol.DelegatedResourceAccountIndex.fromAccounts", 1); + AddressFieldNameMap.put("protocol.DelegatedResourceAccountIndex.toAccounts", 1); + + AddressFieldNameMap.put("protocol.AccountIdentifier.address", 1); + AddressFieldNameMap.put("protocol.TransactionBalanceTrace.Operation.address", 1); + + //***** api.proto ***** + //Return + NameFieldNameMap.put("protocol.Return.message", 1); + //Address + NameFieldNameMap.put("protocol.Address.host", 1); + //Note + NameFieldNameMap.put("protocol.Note.memo", 1); + + //***** Contract.proto ***** + //AccountUpdateContract + NameFieldNameMap.put("protocol.AccountUpdateContract.account_name", 1); + //SetAccountIdContract + NameFieldNameMap.put("protocol.SetAccountIdContract.account_id", 1); + //TransferAssetContract + NameFieldNameMap.put("protocol.TransferAssetContract.asset_name", 1); + //WitnessCreateContract + NameFieldNameMap.put("protocol.WitnessCreateContract.url", 1); + //WitnessUpdateContract + NameFieldNameMap.put("protocol.WitnessUpdateContract.update_url", 1); + //AssetIssueContract + NameFieldNameMap.put("protocol.AssetIssueContract.name", 1); + NameFieldNameMap.put("protocol.AssetIssueContract.abbr", 1); + NameFieldNameMap.put("protocol.AssetIssueContract.description", 1); + NameFieldNameMap.put("protocol.AssetIssueContract.url", 1); + //ParticipateAssetIssueContract + NameFieldNameMap.put("protocol.ParticipateAssetIssueContract.asset_name", 1); + //UpdateAssetContract + NameFieldNameMap.put("protocol.UpdateAssetContract.description", 1); + NameFieldNameMap.put("protocol.UpdateAssetContract.url", 1); + //ExchangeCreateContract + NameFieldNameMap.put("protocol.ExchangeCreateContract.first_token_id", 1); + NameFieldNameMap.put("protocol.ExchangeCreateContract.second_token_id", 1); + //ExchangeInjectContract + NameFieldNameMap.put("protocol.ExchangeInjectContract.token_id", 1); + //ExchangeWithdrawContract + NameFieldNameMap.put("protocol.ExchangeWithdrawContract.token_id", 1); + //ExchangeTransactionContract + NameFieldNameMap.put("protocol.ExchangeTransactionContract.token_id", 1); + + //***** Tron.proto ***** + //AccountId + NameFieldNameMap.put("protocol.AccountId.name", 1); + //Exchange + NameFieldNameMap.put("protocol.Exchange.first_token_id", 1); + NameFieldNameMap.put("protocol.Exchange.second_token_id", 1); + //Account + NameFieldNameMap.put("protocol.Account.account_name", 1); + NameFieldNameMap.put("protocol.Account.asset_issued_name", 1); + NameFieldNameMap.put("protocol.Account.asset_issued_ID", 1); + NameFieldNameMap.put("protocol.Account.account_id", 1); + //authority + NameFieldNameMap.put("protocol.authority.permission_name", 1); + //Transaction + NameFieldNameMap.put("protocol.Transaction.Contract.ContractName", 1); + //TransactionInfo + NameFieldNameMap.put("protocol.TransactionInfo.resMessage", 1); + + //***** market.proto ***** + // MarketSellAssetContract + AddressFieldNameMap.put("protocol.MarketSellAssetContract.owner_address", 1); + NameFieldNameMap.put("protocol.MarketSellAssetContract.sell_token_id", 1); + NameFieldNameMap.put("protocol.MarketSellAssetContract.buy_token_id", 1); + + // MarketCancelOrderContract + AddressFieldNameMap.put("protocol.MarketCancelOrderContract.owner_address", 1); + + // MarketOrder + AddressFieldNameMap.put("protocol.MarketOrder.owner_address", 1); + NameFieldNameMap.put("protocol.MarketOrder.sell_token_id", 1); + NameFieldNameMap.put("protocol.MarketOrder.buy_token_id", 1); + + // MarketOrderPair + NameFieldNameMap.put("protocol.MarketOrderPair.sell_token_id", 1); + NameFieldNameMap.put("protocol.MarketOrderPair.buy_token_id", 1); + + // MarketPriceList + NameFieldNameMap.put("protocol.MarketPriceList.sell_token_id", 1); + NameFieldNameMap.put("protocol.MarketPriceList.buy_token_id", 1); + } + + public static boolean isAddressFormat(final String name) { + return AddressFieldNameMap.containsKey(name); + } + + public static boolean isNameStringFormat(final String name) { + return NameFieldNameMap.containsKey(name); + } +} diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/JsonFormat.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/JsonFormat.java new file mode 100644 index 00000000..1ab5c07c --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/JsonFormat.java @@ -0,0 +1,1663 @@ +package org.tron.plugins.utils; + +/* + Copyright (c) 2009, Orbitz World Wide + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the Orbitz World Wide nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +import com.alibaba.fastjson.JSON; +import com.google.common.collect.ImmutableSet; +import com.google.protobuf.ByteString; +import com.google.protobuf.Descriptors; +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.Descriptors.EnumDescriptor; +import com.google.protobuf.Descriptors.EnumValueDescriptor; +import com.google.protobuf.Descriptors.FieldDescriptor; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; +import com.google.protobuf.UnknownFieldSet; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.CharBuffer; +import java.text.CharacterIterator; +import java.text.StringCharacterIterator; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.Commons; +import org.tron.common.utils.StringUtil; +import org.tron.protos.contract.BalanceContract; + +/** + * Provide ascii text parsing and formatting support for proto2 instances. The implementation + * largely follows google/protobuf/text_format.cc. + * + *

(c) 2009-10 Orbitz World Wide. All Rights Reserved. + * + * @author eliran.bivas@gmail.com Eliran Bivas + * @author aantonov@orbitz.com Alex Antonov

* Based on the original code by: + * @author wenboz@google.com Wenbo Zhu + * @author kenton@google.com Kenton Varda + */ +public class JsonFormat { + + private static final int BUFFER_SIZE = 4096; + private static final Pattern DIGITS = Pattern.compile( + "[0-9]", + Pattern.CASE_INSENSITIVE); + private static final String WRITING_STRING_BUILDER_EXCEPTION + = "Writing to a StringBuilder threw an IOException (should never happen)."; + private static final String EXPECTED_STRING = "Expected string."; + private static final String MISSING_END_QUOTE = "String missing ending quote."; + + public static final boolean ALWAYS_OUTPUT_DEFAULT_VALUE_FIELDS = true; + public static final Set> MESSAGES = ImmutableSet.of( + BalanceContract.AccountBalanceResponse.class, + BalanceContract.BlockBalanceTrace.class, + BalanceContract.TransactionBalanceTrace.Operation.class, + BalanceContract.TransactionBalanceTrace.class + ); + + /** + * Outputs a textual representation of the Protocol Message supplied into the parameter output. + * (This representation is the new version of the classic "ProtocolPrinter" output from the + * original Protocol Buffer system) + */ + public static void print(Message message, Appendable output, boolean selfType) + throws IOException { + JsonGenerator generator = new JsonGenerator(output); + generator.print("{"); + print(message, generator, selfType); + generator.print("}"); + } + + /** + * Outputs a textual representation of {@code fields} to {@code output}. + */ + public static void print(UnknownFieldSet fields, Appendable output, boolean selfType) + throws IOException { + JsonGenerator generator = new JsonGenerator(output); + generator.print("{"); + printUnknownFields(fields, generator, selfType); + generator.print("}"); + } + + protected static void print(Message message, JsonGenerator generator, boolean selfType) + throws IOException { + Map fieldsToPrint = new TreeMap<>(message.getAllFields()); + if (ALWAYS_OUTPUT_DEFAULT_VALUE_FIELDS && MESSAGES.contains(message.getClass())) { + for (FieldDescriptor field : message.getDescriptorForType().getFields()) { + if (field.isOptional()) { + if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE + && !message.hasField(field)) { + // Always skip empty optional message fields. If not we will recurse indefinitely if + // a message has itself as a sub-field. + continue; + } + Descriptors.OneofDescriptor oneof = field.getContainingOneof(); + if (oneof != null && !message.hasField(field)) { + // Skip all oneof fields except the one that is actually set + continue; + } + } + if (!fieldsToPrint.containsKey(field)) { + fieldsToPrint.put(field, message.getField(field)); + } + } + } + + //for (Iterator> iter = message.getAllFields().entrySet() + for (Iterator> iter = fieldsToPrint.entrySet() + .iterator(); iter.hasNext(); ) { + Map.Entry field = iter.next(); + printField(field.getKey(), field.getValue(), generator, selfType); + if (iter.hasNext()) { + generator.print(","); + } + } + + // do not print unknown fields + // if (message.getUnknownFields().asMap().size() > 0) { + // generator.print(", "); + // } + // printUnknownFields(message.getUnknownFields(), generator, selfType); + } + + /** + * Like {@code print()}, but writes directly to a {@code String} and returns it. + */ + public static String printToString(Message message, boolean selfType) { + try { + StringBuilder text = new StringBuilder(); + print(message, text, selfType); + return text.toString(); + } catch (IOException e) { + throw new RuntimeException(WRITING_STRING_BUILDER_EXCEPTION, e); + } + } + + /** + * Like {@code print()}, but writes directly to a {@code String} and returns it. + */ + public static String printToString(Message message) { + try { + StringBuilder text = new StringBuilder(); + print(message, text, true); + return text.toString(); + } catch (IOException e) { + throw new RuntimeException(WRITING_STRING_BUILDER_EXCEPTION, e); + } + } + + /** + * Like {@code print()}, but writes directly to a {@code String} and returns it. + */ + public static String printToString(UnknownFieldSet fields, boolean selfType) { + try { + StringBuilder text = new StringBuilder(); + print(fields, text, selfType); + return text.toString(); + } catch (IOException e) { + throw new RuntimeException(WRITING_STRING_BUILDER_EXCEPTION, e); + } + } + + /** + * Parse a text-format message from {@code input} and merge the contents into {@code builder}. + */ + public static void merge(Readable input, Message.Builder builder) throws IOException { + merge(input, ExtensionRegistry.getEmptyRegistry(), builder, true); + } + + /** + * Parse a text-format message from {@code input} and merge the contents into {@code builder}. + */ + public static void merge(CharSequence input, Message.Builder builder) throws ParseException { + merge(input, ExtensionRegistry.getEmptyRegistry(), builder, true); + } + + /** + * Parse a text-format message from {@code input} and merge the contents into {@code builder}. + */ + public static void merge(Readable input, Message.Builder builder, boolean selfType) + throws IOException { + merge(input, ExtensionRegistry.getEmptyRegistry(), builder, selfType); + } + + /** + * Parse a text-format message from {@code input} and merge the contents into {@code builder}. + */ + public static void merge(CharSequence input, Message.Builder builder, boolean selfType) + throws ParseException { + merge(input, ExtensionRegistry.getEmptyRegistry(), builder, selfType); + } + + /** + * Parse a text-format message from {@code input} and merge the contents into {@code builder}. + * Extensions will be recognized if they are registered in {@code extensionRegistry}. + */ + public static void merge(Readable input, + ExtensionRegistry extensionRegistry, + Message.Builder builder, boolean selfType) throws IOException { + // Read the entire input to a String then parse that. + + // If StreamTokenizer were not quite so crippled, or if there were a kind + // of Reader that could read in chunks that match some particular regex, + // or if we wanted to write a custom Reader to tokenize our stream, then + // we would not have to read to one big String. Alas, none of these is + // the case. Oh well. + + merge(toStringBuilder(input), extensionRegistry, builder, selfType); + } + + /** + * Parse a text-format message from {@code input} and merge the contents into {@code builder}. + * Extensions will be recognized if they are registered in {@code extensionRegistry}. + */ + public static void merge(CharSequence input, + ExtensionRegistry extensionRegistry, + Message.Builder builder, boolean selfType) throws ParseException { + Tokenizer tokenizer = new Tokenizer(input); + + // Based on the state machine @ http://json.org/ + + tokenizer.consume("{"); // Needs to happen when the object starts. + while (!tokenizer.tryConsume("}")) { // Continue till the object is done + mergeField(tokenizer, extensionRegistry, builder, selfType); + } + // Test to make sure the tokenizer has reached the end of the stream. + if (!tokenizer.atEnd()) { + throw tokenizer.parseException( + "Expecting the end of the stream, but there seems to be more data! " + + "Check the input for a valid JSON format."); + } + } + + public static String printErrorMsg(Exception ex) { + StringBuilder text = new StringBuilder(); + text.append("{"); + text.append("\"Error\":"); + text.append("\""); + text.append(ex.getMessage()); + text.append("\""); + text.append("}"); + return text.toString(); + } + + public static void printField(FieldDescriptor field, Object value, JsonGenerator generator, + boolean selfType) + throws IOException { + + printSingleField(field, value, generator, selfType); + } + + private static void printSingleField(FieldDescriptor field, + Object value, JsonGenerator generator, boolean selfType) throws IOException { + if (field.isExtension()) { + generator.print("\""); + // We special-case MessageSet elements for compatibility with proto1. + if (field.getContainingType().getOptions().getMessageSetWireFormat() + && (field.getType() == FieldDescriptor.Type.MESSAGE) && (field.isOptional()) + // object equality + && (field.getExtensionScope() == field.getMessageType())) { + generator.print(field.getMessageType().getFullName()); + } else { + generator.print(field.getFullName()); + } + generator.print("\""); + } else { + generator.print("\""); + if (field.getType() == FieldDescriptor.Type.GROUP) { + // Groups must be serialized with their original capitalization. + generator.print(field.getMessageType().getName()); + } else { + generator.print(field.getName()); + } + generator.print("\""); + } + + // Done with the name, on to the value + + if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { + generator.print(": "); + generator.indent(); + } else { + generator.print(": "); + } + + if (field.isRepeated()) { + // Repeated field. Print each element. + generator.print("["); + for (Iterator iter = ((List) value).iterator(); iter.hasNext(); ) { + printFieldValue(field, iter.next(), generator, selfType); + if (iter.hasNext()) { + generator.print(","); + } + } + generator.print("]"); + } else { + printFieldValue(field, value, generator, selfType); + if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { + generator.outdent(); + } + } + } + + private static void printFieldValue(FieldDescriptor field, Object value, + JsonGenerator generator, boolean selfType) + throws IOException { + switch (field.getType()) { + case INT32: + case INT64: + case SINT32: + case SINT64: + case SFIXED32: + case SFIXED64: + case FLOAT: + case DOUBLE: + case BOOL: + // Good old toString() does what we want for these types. + generator.print(value.toString()); + break; + + case UINT32: + case FIXED32: + generator.print(unsignedToString((Integer) value)); + break; + + case UINT64: + case FIXED64: + generator.print(unsignedToString((Long) value)); + break; + + case STRING: + generator.print("\""); + generator.print(escapeText((String) value)); + generator.print("\""); + break; + + case BYTES: { + generator.print("\""); + generator.print(escapeBytes((ByteString) value, field.getFullName(), selfType)); + generator.print("\""); + break; + } + + case ENUM: { + generator.print("\""); + generator.print(((EnumValueDescriptor) value).getName()); + generator.print("\""); + break; + } + + case MESSAGE: + case GROUP: + generator.print("{"); + print((Message) value, generator, selfType); + generator.print("}"); + break; + default: + } + } + + protected static void printUnknownFields(UnknownFieldSet unknownFields, JsonGenerator generator, + boolean selfType) throws IOException { + boolean firstField = true; + for (Map.Entry entry : unknownFields.asMap().entrySet()) { + final UnknownFieldSet.Field field = entry.getValue(); + if (firstField) { + firstField = false; + } else { + generator.print(", "); + } + + generator.print("\""); + generator.print(entry.getKey().toString()); + generator.print("\""); + generator.print(": ["); + + boolean firstValue = true; + for (long value : field.getVarintList()) { + if (firstValue) { + firstValue = false; + } else { + generator.print(", "); + } + generator.print(unsignedToString(value)); + } + for (int value : field.getFixed32List()) { + if (firstValue) { + firstValue = false; + } else { + generator.print(", "); + } + generator.print("\""); + generator.print(String.format((Locale) null, "0x%08x", value)); + generator.print("\""); + } + for (long value : field.getFixed64List()) { + if (firstValue) { + firstValue = false; + } else { + generator.print(", "); + } + generator.print("\""); + generator.print(String.format((Locale) null, "0x%016x", value)); + generator.print("\""); + } + for (ByteString value : field.getLengthDelimitedList()) { + if (firstValue) { + firstValue = false; + } else { + generator.print(", "); + } + generator.print("\""); + generator.print(escapeBytes(value, "Hex", selfType)); //Just to HEX + generator.print("\""); + } + for (UnknownFieldSet value : field.getGroupList()) { + if (firstValue) { + firstValue = false; + } else { + generator.print(", "); + } + generator.print("{"); + printUnknownFields(value, generator, selfType); + generator.print("}"); + } + generator.print("]"); + } + } + + /** + * Convert an unsigned 32-bit integer to a string. + */ + private static String unsignedToString(int value) { + if (value >= 0) { + return Integer.toString(value); + } else { + return Long.toString((value) & 0x00000000FFFFFFFFL); + } + } + + // ================================================================= + // Parsing + + /** + * Convert an unsigned 64-bit integer to a string. + */ + private static String unsignedToString(long value) { + if (value >= 0) { + return Long.toString(value); + } else { + // Pull off the most-significant bit so that BigInteger doesn't think + // the number is negative, then set it again using setBit(). + return BigInteger.valueOf(value & 0x7FFFFFFFFFFFFFFFL).setBit(63).toString(); + } + } + + + // TODO(chrisn): See if working around java.io.Reader#read(CharBuffer) + // overhead is worthwhile + protected static StringBuilder toStringBuilder(Readable input) throws IOException { + StringBuilder text = new StringBuilder(); + CharBuffer buffer = CharBuffer.allocate(BUFFER_SIZE); + while (true) { + int n = input.read(buffer); + if (n == -1) { + break; + } + buffer.flip(); + text.append(buffer, 0, n); + } + return text; + } + + /** + * Parse a single field from {@code tokenizer} and merge it into {@code builder}. If a ',' is + * detected after the field ends, the next field will be parsed automatically + */ + protected static void mergeField(Tokenizer tokenizer, + ExtensionRegistry extensionRegistry, Message.Builder builder, + boolean selfType) throws ParseException { + FieldDescriptor field; + Descriptor type = builder.getDescriptorForType(); + final ExtensionRegistry.ExtensionInfo extension; + boolean unknown = false; + + String name = tokenizer.consumeIdentifier(); + field = type.findFieldByName(name); + + // Group names are expected to be capitalized as they appear in the + // .proto file, which actually matches their type names, not their field + // names. + if (field == null) { + // Explicitly specify US locale so that this code does not break when + // executing in Turkey. + String lowerName = name.toLowerCase(Locale.US); + field = type.findFieldByName(lowerName); + // If the case-insensitive match worked but the field is NOT a group, + if ((field != null) && (field.getType() != FieldDescriptor.Type.GROUP)) { + field = null; + } + } + // Again, special-case group names as described above. + if ((field != null) && (field.getType() == FieldDescriptor.Type.GROUP) + && !field.getMessageType().getName().equals(name)) { + field = null; + } + + // Last try to lookup by field-index if 'name' is numeric, + // which indicates a possible unknown field + if (field == null && DIGITS.matcher(name).matches()) { + field = type.findFieldByNumber(Integer.parseInt(name)); + unknown = true; + } + + // Finally, look for extensions + extension = extensionRegistry.findExtensionByName(name); + if (extension != null) { + if (extension.descriptor.getContainingType() != type) { + throw tokenizer.parseExceptionPreviousToken("Extension \"" + name + + "\" does not extend message type \"" + + type.getFullName() + "\"."); + } + field = extension.descriptor; + } + + // Disabled throwing exception if field not found, since it could be a different version. + if (field == null) { + handleMissingField(tokenizer, extensionRegistry, builder); + //throw tokenizer.parseExceptionPreviousToken("Message type \"" + type.getFullName() + // + "\" has no field named \"" + name + // + "\"."); + } + + if (field != null) { + tokenizer.consume(":"); + boolean array = tokenizer.tryConsume("["); + + if (array) { + while (!tokenizer.tryConsume("]")) { + handleValue(tokenizer, extensionRegistry, builder, field, extension, unknown, selfType); + tokenizer.tryConsume(","); + } + } else { + handleValue(tokenizer, extensionRegistry, builder, field, extension, unknown, selfType); + } + } + + if (tokenizer.tryConsume(",")) { + // Continue with the next field + mergeField(tokenizer, extensionRegistry, builder, selfType); + } + } + + private static void handleMissingField(Tokenizer tokenizer, + ExtensionRegistry extensionRegistry, + Message.Builder builder) throws ParseException { + tokenizer.tryConsume(":"); + if ("{".equals(tokenizer.currentToken())) { + // Message structure + tokenizer.consume("{"); + do { + tokenizer.consumeIdentifier(); + handleMissingField(tokenizer, extensionRegistry, builder); + } while (tokenizer.tryConsume(",")); + tokenizer.consume("}"); + } else if ("[".equals(tokenizer.currentToken())) { + // Collection + tokenizer.consume("["); + do { + handleMissingField(tokenizer, extensionRegistry, builder); + } while (tokenizer.tryConsume(",")); + tokenizer.consume("]"); + } else { //if (!",".equals(tokenizer.currentToken)){ + // Primitive value + if ("null".equals(tokenizer.currentToken())) { + tokenizer.consume("null"); + } else if (tokenizer.lookingAtInteger()) { + tokenizer.consumeInt64(); + } else if (tokenizer.lookingAtBoolean()) { + tokenizer.consumeBoolean(); + } else { + tokenizer.consumeString(); + } + } + } + + private static void handleValue(Tokenizer tokenizer, + ExtensionRegistry extensionRegistry, + Message.Builder builder, + FieldDescriptor field, + ExtensionRegistry.ExtensionInfo extension, + boolean unknown, boolean selfType) throws ParseException { + + Object value = null; + if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { + value = handleObject(tokenizer, extensionRegistry, builder, field, extension, unknown, + selfType); + } else { + value = handlePrimitive(tokenizer, field, selfType); + } + if (value != null) { + if (field.isRepeated()) { + builder.addRepeatedField(field, value); + } else { + builder.setField(field, value); + } + } + } + + private static Object handlePrimitive(Tokenizer tokenizer, FieldDescriptor field, + boolean selfType) + throws ParseException { + Object value = null; + if ("null".equals(tokenizer.currentToken())) { + tokenizer.consume("null"); + return value; + } + switch (field.getType()) { + case INT32: + case SINT32: + case SFIXED32: + value = tokenizer.consumeInt32(); + break; + + case INT64: + case SINT64: + case SFIXED64: + value = tokenizer.consumeInt64(); + break; + + case UINT32: + case FIXED32: + value = tokenizer.consumeUInt32(); + break; + + case UINT64: + case FIXED64: + value = tokenizer.consumeUInt64(); + break; + + case FLOAT: + value = tokenizer.consumeFloat(); + break; + + case DOUBLE: + value = tokenizer.consumeDouble(); + break; + + case BOOL: + value = tokenizer.consumeBoolean(); + break; + + case STRING: + value = tokenizer.consumeString(); + break; + + case BYTES: + value = tokenizer.consumeByteString(field.getFullName(), selfType); + break; + + case ENUM: { + EnumDescriptor enumType = field.getEnumType(); + + if (tokenizer.lookingAtInteger()) { + int number = tokenizer.consumeInt32(); + value = enumType.findValueByNumber(number); + if (value == null) { + throw tokenizer.parseExceptionPreviousToken("Enum type \"" + + enumType.getFullName() + + "\" has no value with number " + + number + "."); + } + } else { + String id = tokenizer.consumeIdentifier(); + if (StringUtils.isAllLowerCase(id)) { + char b = id.charAt(0); + b = (char) (b + 'A' - 'a'); + String s = id.substring(1); + id = b + s; + } + value = enumType.findValueByName(id); + if (value == null) { + throw tokenizer.parseExceptionPreviousToken("Enum type \"" + + enumType.getFullName() + + "\" has no value named \"" + + id + "\"."); + } + } + + break; + } + + case MESSAGE: + case GROUP: + throw new RuntimeException("Can't get here."); + default: + } + return value; + } + + private static Object handleObject(Tokenizer tokenizer, + ExtensionRegistry extensionRegistry, + Message.Builder builder, + FieldDescriptor field, + ExtensionRegistry.ExtensionInfo extension, + boolean unknown, boolean selfType) throws ParseException { + + Message.Builder subBuilder; + if (extension == null) { + subBuilder = builder.newBuilderForField(field); + } else { + subBuilder = extension.defaultInstance.newBuilderForType(); + } + + if (unknown) { + ByteString data = tokenizer.consumeByteString("", selfType); + try { + subBuilder.mergeFrom(data); + return subBuilder.build(); + } catch (InvalidProtocolBufferException e) { + throw tokenizer.parseException("Failed to build " + field.getFullName() + " from " + data); + } + } + + tokenizer.consume("{"); + String endToken = "}"; + + while (!tokenizer.tryConsume(endToken)) { + if (tokenizer.atEnd()) { + throw tokenizer.parseException("Expected \"" + endToken + "\"."); + } + mergeField(tokenizer, extensionRegistry, subBuilder, selfType); + if (tokenizer.tryConsume(",")) { + // there are more fields in the object, so continue + continue; + } + } + + return subBuilder.build(); + } + + /** + * Escapes bytes in the format used in protocol buffer text format, which is the same as the + * format used for C string literals. All bytes that are not printable 7-bit ASCII characters are + * escaped, as well as backslash, single-quote, and double-quote characters. Characters for which + * no defined short-hand escape sequence is defined will be escaped using 3-digit octal + * sequences. + */ + static String escapeBytes(ByteString input) { + return ByteArray.toHexString(input.toByteArray()); + } + + static String escapeBytes(ByteString input, final String fliedName, boolean selfType) { + if (!selfType) { + return ByteArray.toHexString(input.toByteArray()); + } else { + return escapeBytesSelfType(input, fliedName); + } + } + + static String escapeBytesSelfType(ByteString input, final String fliedName) { + //Address + if (HttpSelfFormatFieldName.isAddressFormat(fliedName)) { + return StringUtil.encode58Check(input.toByteArray()); + } + //Normal String + if (HttpSelfFormatFieldName.isNameStringFormat(fliedName)) { + String result = new String(input.toByteArray()); + result = result.replaceAll("\"", "\\\\\""); + try { + JSON.parseObject("{\"key\":\"" + result + "\"}"); + return result; + } catch (Exception e) { + return ByteArray.toHexString(input.toByteArray()); + } + } + //HEX + return ByteArray.toHexString(input.toByteArray()); + } + + static String unicodeEscaped(char ch) { + if (ch < 0x10) { + return "\\u000" + Integer.toHexString(ch); + } else if (ch < 0x100) { + return "\\u00" + Integer.toHexString(ch); + } else if (ch < 0x1000) { + return "\\u0" + Integer.toHexString(ch); + } + return "\\u" + Integer.toHexString(ch); + } + + static ByteString unescapeBytes(CharSequence input) throws InvalidEscapeSequence { + try { + return ByteString.copyFrom(ByteArray.fromHexString(input.toString())); + } catch (Exception e) { + throw new InvalidEscapeSequence("invalidate hex String"); + } + } + + // ================================================================= + // Utility functions + // + // Some of these methods are package-private because Descriptors.java uses + // them. + + /** + * Implements JSON string escaping as specified here. + *

  • The following characters are escaped by prefixing them with a '\' : + * \b,\f,\n,\r,\t,\,"
  • Other control characters in the range 0x0000-0x001F are escaped + * using the \\uXXXX notation
  • UTF-16 surrogate pairs are encoded using the \\uXXXX\\uXXXX + * notation
  • any other character is printed as-is
+ */ + static String escapeText(String input) { + StringBuilder builder = new StringBuilder(input.length()); + CharacterIterator iter = new StringCharacterIterator(input); + for (char c = iter.first(); c != CharacterIterator.DONE; c = iter.next()) { + switch (c) { + case '\b': + builder.append("\\b"); + break; + case '\f': + builder.append("\\f"); + break; + case '\n': + builder.append("\\n"); + break; + case '\r': + builder.append("\\r"); + break; + case '\t': + builder.append("\\t"); + break; + case '\\': + builder.append("\\\\"); + break; + case '"': + builder.append("\\\""); + break; + default: + // Check for other control characters + if (c >= 0x0000 && c <= 0x001F) { + appendEscapedUnicode(builder, c); + } else if (Character.isHighSurrogate(c)) { + // Encode the surrogate pair using 2 six-character sequence (\\uXXXX\\uXXXX) + appendEscapedUnicode(builder, c); + c = iter.next(); + if (c == CharacterIterator.DONE) { + throw new IllegalArgumentException( + "invalid unicode string: unexpected high surrogate pair value " + + "without corresponding low value."); + } + appendEscapedUnicode(builder, c); + } else { + // Anything else can be printed as-is + builder.append(c); + } + break; + } + } + return builder.toString(); + } + + static void appendEscapedUnicode(StringBuilder builder, char ch) { + String prefix = "\\u"; + if (ch < 0x10) { + prefix = "\\u000"; + } else if (ch < 0x100) { + prefix = "\\u00"; + } else if (ch < 0x1000) { + prefix = "\\u0"; + } + builder.append(prefix).append(Integer.toHexString(ch)); + } + + /** + * Un-escape a text string as escaped using {@link #escapeText(String)}. + */ + static String unescapeText(String input) throws InvalidEscapeSequence { + StringBuilder builder = new StringBuilder(); + char[] array = input.toCharArray(); + for (int i = 0; i < array.length; i++) { + char c = array[i]; + if (c == '\\') { + if (i + 1 < array.length) { + ++i; + c = array[i]; + switch (c) { + case 'b': + builder.append('\b'); + break; + case 'f': + builder.append('\f'); + break; + case 'n': + builder.append('\n'); + break; + case 'r': + builder.append('\r'); + break; + case 't': + builder.append('\t'); + break; + case '\\': + builder.append('\\'); + break; + case '"': + builder.append('\"'); + break; + case '\'': + builder.append('\''); + break; + case 'u': + // read the next 4 chars + if (i + 4 < array.length) { + ++i; + int code = Integer.parseInt(new String(array, i, 4), 16); + // this cast is safe because we know how many chars we read + builder.append((char) code); + i += 3; + } else { + throw new InvalidEscapeSequence("Invalid escape sequence: '\\u' at end of string."); + } + break; + default: + throw new InvalidEscapeSequence("Invalid escape sequence: '\\" + c + "'"); + } + } else { + throw new InvalidEscapeSequence("Invalid escape sequence: '\\' at end of string."); + } + } else { + builder.append(c); + } + } + + return builder.toString(); + } + + /** + * Is this an octal digit. + */ + private static boolean isOctal(char c) { + return ('0' <= c) && (c <= '7'); + } + + /** + * Is this a hex digit. + */ + private static boolean isHex(char c) { + return (('0' <= c) && (c <= '9')) || (('a' <= c) && (c <= 'f')) + || (('A' <= c) && (c <= 'F')); + } + + /** + * Interpret a character as a digit (in any base up to 36) and return the numeric value. This is + * like {@code Character.digit()} but we don't accept non-ASCII digits. + */ + private static int digitValue(char c) { + if (('0' <= c) && (c <= '9')) { + return c - '0'; + } else if (('a' <= c) && (c <= 'z')) { + return c - 'a' + 10; + } else { + return c - 'A' + 10; + } + } + + /** + * Parse a 32-bit signed integer from the text. Unlike the Java standard {@code + * Integer.parseInt()}, this function recognizes the prefixes "0x" and "0" to signify hexidecimal + * and octal numbers, respectively. + */ + static int parseInt32(String text) throws NumberFormatException { + return (int) parseInteger(text, true, false); + } + + /** + * Parse a 32-bit unsigned integer from the text. Unlike the Java standard {@code + * Integer.parseInt()}, this function recognizes the prefixes "0x" and "0" to signify hexidecimal + * and octal numbers, respectively. The result is coerced to a (signed) {@code int} when returned + * since Java has no unsigned integer type. + */ + static int parseUInt32(String text) throws NumberFormatException { + return (int) parseInteger(text, false, false); + } + + /** + * Parse a 64-bit signed integer from the text. Unlike the Java standard {@code + * Integer.parseInt()}, this function recognizes the prefixes "0x" and "0" to signify hexidecimal + * and octal numbers, respectively. + */ + static long parseInt64(String text) throws NumberFormatException { + return parseInteger(text, true, true); + } + + /** + * Parse a 64-bit unsigned integer from the text. Unlike the Java standard {@code + * Integer.parseInt()}, this function recognizes the prefixes "0x" and "0" to signify hexidecimal + * and octal numbers, respectively. The result is coerced to a (signed) {@code long} when returned + * since Java has no unsigned long type. + */ + static long parseUInt64(String text) throws NumberFormatException { + return parseInteger(text, false, true); + } + + private static long parseInteger(String text, boolean isSigned, boolean isLong) + throws NumberFormatException { + int pos = 0; + + boolean negative = false; + if (text.startsWith("-", pos)) { + if (!isSigned) { + throw new NumberFormatException("Number must be positive: " + text); + } + ++pos; + negative = true; + } + + int radix = 10; + if (text.startsWith("0x", pos)) { + pos += 2; + radix = 16; + } else if (text.startsWith("0", pos)) { + radix = 8; + } + + String numberText = text.substring(pos); + + long result = 0; + if (numberText.length() < 16) { + // Can safely assume no overflow. + result = Long.parseLong(numberText, radix); + if (negative) { + result = -result; + } + + // Check bounds. + // No need to check for 64-bit numbers since they'd have to be 16 chars + // or longer to overflow. + if (!isLong) { + if (isSigned) { + if ((result > Integer.MAX_VALUE) || (result < Integer.MIN_VALUE)) { + throw new NumberFormatException("Number out of range for 32-bit signed integer: " + + text); + } + } else { + if ((result >= (1L << 32)) || (result < 0)) { + throw new NumberFormatException("Number out of range for 32-bit unsigned integer: " + + text); + } + } + } + } else { + BigInteger bigValue = new BigInteger(numberText, radix); + if (negative) { + bigValue = bigValue.negate(); + } + + // Check bounds. + if (!isLong) { + if (isSigned) { + if (bigValue.bitLength() > 31) { + throw new NumberFormatException("Number out of range for 32-bit signed integer: " + + text); + } + } else { + if (bigValue.bitLength() > 32) { + throw new NumberFormatException("Number out of range for 32-bit unsigned integer: " + + text); + } + } + } else { + if (isSigned) { + if (bigValue.bitLength() > 63) { + throw new NumberFormatException("Number out of range for 64-bit signed integer: " + + text); + } + } else { + if (bigValue.bitLength() > 64) { + throw new NumberFormatException("Number out of range for 64-bit unsigned integer: " + + text); + } + } + } + + result = bigValue.longValueExact(); + } + + return result; + } + + /** + * An inner class for writing text to the output stream. + */ + protected static class JsonGenerator { + + private Appendable output; + private boolean atStartOfLine = true; + private StringBuilder indent = new StringBuilder(); + + public JsonGenerator(Appendable output) { + this.output = output; + } + + /** + * Indent text by two spaces. After calling Indent(), two spaces will be inserted at the + * beginning of each line of text. Indent() may be called multiple times to produce deeper + * indents. + */ + public void indent() { + indent.append(" "); + } + + /** + * Reduces the current indent level by two spaces, or crashes if the indent level is zero. + */ + public void outdent() { + int length = indent.length(); + if (length == 0) { + throw new IllegalArgumentException(" Outdent() without matching Indent()."); + } + indent.delete(length - 2, length); + } + + /** + * Print text to the output stream. + */ + public void print(CharSequence text) throws IOException { + int size = text.length(); + int pos = 0; + + for (int i = 0; i < size; i++) { + if (text.charAt(i) == '\n') { + write(text.subSequence(pos, size), i - pos + 1); + pos = i + 1; + atStartOfLine = true; + } + } + write(text.subSequence(pos, size), size - pos); + } + + private void write(CharSequence data, int size) throws IOException { + if (size == 0) { + return; + } + if (atStartOfLine) { + atStartOfLine = false; + output.append(indent); + } + output.append(data); + } + } + + /** + * Represents a stream of tokens parsed from a {@code String}. + *

+ * + *

The Java standard library provides many classes that you might think would be useful for + * implementing this, but aren't. For example: + *

+ *

  • {@code java.io.StreamTokenizer}: This almost does what we want -- or, at least, + * something that would get us close to what we want -- except for one fatal flaw: It + * automatically un-escapes strings using Java escape sequences, which do not include all the + * escape sequences we need to support (e.g. '\x').
  • {@code java.util.Scanner}: This seems like + * a great way at least to parse regular expressions out of a stream (so we wouldn't have to load + * the entire input into a single string before parsing). Sadly, {@code Scanner} requires that + * tokens be delimited with some delimiter. Thus, although the text "foo:" should parse to two + * tokens ("foo" and ":"), {@code Scanner} would recognize it only as a single token. Furthermore, + * {@code Scanner} provides no way to inspect the contents of delimiters, making it impossible to + * keep track of line and column numbers.
+ *

+ * + *

Luckily, Java's regular expression support does manage to be useful to us. (Barely: We need + * {@code Matcher.usePattern()}, which is new in Java 1.5.) So, we can use that, at least. + * Unfortunately, this implies that we need to have the entire input in one contiguous string. + */ + protected static class Tokenizer { + + // We use possesive quantifiers (*+ and ++) because otherwise the Java + // regex matcher has stack overflows on large inputs. + private static final Pattern WHITESPACE = + Pattern.compile("(\\s|(#.*$))++", Pattern.MULTILINE); + private static final Pattern TOKEN = Pattern.compile( + "[a-zA-Z_][0-9a-zA-Z_+-]*+|" + // an identifier + "[.]?[0-9+-][0-9a-zA-Z_.+-]*+|" + // a number + "\"([^\"\n\\\\]|\\\\.)*+(\"|\\\\?$)|" + // a double-quoted string + "\'([^\'\n\\\\]|\\\\.)*+(\'|\\\\?$)", // a single-quoted string + Pattern.MULTILINE); + private static final Pattern DOUBLE_INFINITY = Pattern.compile( + "-?inf(inity)?", + Pattern.CASE_INSENSITIVE); + private static final Pattern FLOAT_INFINITY = Pattern.compile( + "-?inf(inity)?f?", + Pattern.CASE_INSENSITIVE); + private static final Pattern FLOAT_NAN = Pattern.compile( + "nanf?", + Pattern.CASE_INSENSITIVE); + private final CharSequence text; + private final Matcher matcher; + private String currentToken; + // The character index within this.text at which the current token begins. + private int pos = 0; + // The line and column numbers of the current token. + private int line = 0; + private int column = 0; + // The line and column numbers of the previous token (allows throwing + // errors *after* consuming). + private int previousLine = 0; + private int previousColumn = 0; + + /** + * Construct a tokenizer that parses tokens from the given text. + */ + public Tokenizer(CharSequence text) { + this.text = text; + matcher = WHITESPACE.matcher(text); + skipWhitespace(); + nextToken(); + } + + static ByteString unescapeBytes(CharSequence input) throws InvalidEscapeSequence { + try { + return ByteString.copyFrom(ByteArray.fromHexString(input.toString())); + } catch (Exception e) { + throw new InvalidEscapeSequence("INVALID hex String"); + } + } + + static ByteString unescapeBytesSelfType(String input, final String fliedName) + throws InvalidEscapeSequence { + //Address base58 -> ByteString + if (HttpSelfFormatFieldName.isAddressFormat(fliedName)) { + return ByteString.copyFrom(Commons.decodeFromBase58Check(input)); + } + + //Normal String -> ByteString + if (HttpSelfFormatFieldName.isNameStringFormat(fliedName)) { + return ByteString.copyFromUtf8(input); + } + + return unescapeBytes(input); + } + + /** + * Are we at the end of the input. + */ + public boolean atEnd() { + return currentToken.length() == 0; + } + + /** + * Advance to the next token. + */ + public void nextToken() { + previousLine = line; + previousColumn = column; + + // Advance the line counter to the current position. + while (pos < matcher.regionStart()) { + if (text.charAt(pos) == '\n') { + ++line; + column = 0; + } else { + ++column; + } + ++pos; + } + + // Match the next token. + if (matcher.regionStart() == matcher.regionEnd()) { + // EOF + currentToken = ""; + } else { + matcher.usePattern(TOKEN); + if (matcher.lookingAt()) { + currentToken = matcher.group(); + matcher.region(matcher.end(), matcher.regionEnd()); + } else { + // Take one character. + currentToken = String.valueOf(text.charAt(pos)); + matcher.region(pos + 1, matcher.regionEnd()); + } + + skipWhitespace(); + } + } + + /** + * Skip over any whitespace so that the matcher region starts at the next token. + */ + private void skipWhitespace() { + matcher.usePattern(WHITESPACE); + if (matcher.lookingAt()) { + matcher.region(matcher.end(), matcher.regionEnd()); + } + } + + /** + * If the next token exactly matches {@code token}, consume it and return {@code true}. + * Otherwise, return {@code false} without doing anything. + */ + public boolean tryConsume(String token) { + if (currentToken.equals(token)) { + nextToken(); + return true; + } else { + return false; + } + } + + /** + * If the next token exactly matches {@code token}, consume it. Otherwise, throw a {@link + * ParseException}. + */ + public void consume(String token) throws ParseException { + if (!tryConsume(token)) { + throw parseException("Expected \"" + token + "\"."); + } + } + + /** + * Returns {@code true} if the next token is an integer, but does not consume it. + */ + public boolean lookingAtInteger() { + if (currentToken.length() == 0) { + return false; + } + + char c = currentToken.charAt(0); + return (('0' <= c) && (c <= '9')) || (c == '-') || (c == '+'); + } + + /** + * Returns {@code true} if the next token is a boolean (true/false), but does not consume it. + */ + public boolean lookingAtBoolean() { + if (currentToken.length() == 0) { + return false; + } + + return ("true".equals(currentToken) || "false".equals(currentToken)); + } + + /** + * @return currentToken to which the Tokenizer is pointing. + */ + public String currentToken() { + return currentToken; + } + + /** + * If the next token is an identifier, consume it and return its value. Otherwise, throw a + * {@link ParseException}. + */ + public String consumeIdentifier() throws ParseException { + for (int i = 0; i < currentToken.length(); i++) { + char c = currentToken.charAt(i); + if ((('a' <= c) && (c <= 'z')) || (('A' <= c) && (c <= 'Z')) + || (('0' <= c) && (c <= '9')) || (c == '_') || (c == '.') || (c == '"')) { + // OK + } else { + throw parseException("Expected identifier. -" + c); + } + } + + String result = currentToken; + // Need to clean-up result to remove quotes of any kind + result = result.replaceAll("\"|'", ""); + nextToken(); + return result; + } + + /** + * If the next token is a 32-bit signed integer, consume it and return its value. Otherwise, + * throw a {@link ParseException}. + */ + public int consumeInt32() throws ParseException { + try { + int result = parseInt32(currentToken); + nextToken(); + return result; + } catch (NumberFormatException e) { + throw integerParseException(e); + } + } + + /** + * If the next token is a 32-bit unsigned integer, consume it and return its value. Otherwise, + * throw a {@link ParseException}. + */ + public int consumeUInt32() throws ParseException { + try { + int result = parseUInt32(currentToken); + nextToken(); + return result; + } catch (NumberFormatException e) { + throw integerParseException(e); + } + } + + /** + * If the next token is a 64-bit signed integer, consume it and return its value. Otherwise, + * throw a {@link ParseException}. + */ + public long consumeInt64() throws ParseException { + try { + long result = parseInt64(currentToken); + nextToken(); + return result; + } catch (NumberFormatException e) { + throw integerParseException(e); + } + } + + /** + * If the next token is a 64-bit unsigned integer, consume it and return its value. Otherwise, + * throw a {@link ParseException}. + */ + public long consumeUInt64() throws ParseException { + try { + long result = parseUInt64(currentToken); + nextToken(); + return result; + } catch (NumberFormatException e) { + throw integerParseException(e); + } + } + + /** + * If the next token is a double, consume it and return its value. Otherwise, throw a {@link + * ParseException}. + */ + public double consumeDouble() throws ParseException { + // We need to parse infinity and nan separately because + // Double.parseDouble() does not accept "inf", "infinity", or "nan". + if (DOUBLE_INFINITY.matcher(currentToken).matches()) { + boolean negative = currentToken.startsWith("-"); + nextToken(); + return negative ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY; + } + if (currentToken.equalsIgnoreCase("nan")) { + nextToken(); + return Double.NaN; + } + try { + double result = Double.parseDouble(currentToken); + nextToken(); + return result; + } catch (NumberFormatException e) { + throw floatParseException(e); + } + } + + /** + * If the next token is a float, consume it and return its value. Otherwise, throw a {@link + * ParseException}. + */ + public float consumeFloat() throws ParseException { + // We need to parse infinity and nan separately because + // Float.parseFloat() does not accept "inf", "infinity", or "nan". + if (FLOAT_INFINITY.matcher(currentToken).matches()) { + boolean negative = currentToken.startsWith("-"); + nextToken(); + return negative ? Float.NEGATIVE_INFINITY : Float.POSITIVE_INFINITY; + } + if (FLOAT_NAN.matcher(currentToken).matches()) { + nextToken(); + return Float.NaN; + } + try { + float result = Float.parseFloat(currentToken); + nextToken(); + return result; + } catch (NumberFormatException e) { + throw floatParseException(e); + } + } + + /** + * If the next token is a boolean, consume it and return its value. Otherwise, throw a {@link + * ParseException}. + */ + public boolean consumeBoolean() throws ParseException { + if (currentToken.equals("true")) { + nextToken(); + return true; + } else if (currentToken.equals("false")) { + nextToken(); + return false; + } else { + throw parseException("Expected \"true\" or \"false\"."); + } + } + + /** + * If the next token is a string, consume it and return its (unescaped) value. Otherwise, throw + * a {@link ParseException}. + */ + public String consumeString() throws ParseException { + char quote = currentToken.length() > 0 ? currentToken.charAt(0) : '\0'; + if ((quote != '\"') && (quote != '\'')) { + throw parseException(EXPECTED_STRING); + } + + if ((currentToken.length() < 2) + || (currentToken.charAt(currentToken.length() - 1) != quote)) { + throw parseException(MISSING_END_QUOTE); + } + + try { + String escaped = currentToken.substring(1, currentToken.length() - 1); + String result = unescapeText(escaped); + nextToken(); + return result; + } catch (InvalidEscapeSequence e) { + throw parseException(e.getMessage()); + } + } + + public ByteString consumeByteString() throws ParseException { + char quote = currentToken.length() > 0 ? currentToken.charAt(0) : '\0'; + if ((quote != '\"') && (quote != '\'')) { + throw parseException(EXPECTED_STRING); + } + + if ((currentToken.length() < 2) + || (currentToken.charAt(currentToken.length() - 1) != quote)) { + throw parseException(MISSING_END_QUOTE); + } + + try { + String escaped = currentToken.substring(1, currentToken.length() - 1); + ByteString result = unescapeBytes(escaped); + nextToken(); + return result; + } catch (InvalidEscapeSequence e) { + throw parseException(e.getMessage()); + } + } + + public ByteString consumeByteString(final String fieldName, boolean selfType) + throws ParseException { + char quote = currentToken.length() > 0 ? currentToken.charAt(0) : '\0'; + if ((quote != '\"') && (quote != '\'')) { + throw parseException(EXPECTED_STRING); + } + + if ((currentToken.length() < 2) + || (currentToken.charAt(currentToken.length() - 1) != quote)) { + throw parseException(MISSING_END_QUOTE); + } + + try { + String escaped = currentToken.substring(1, currentToken.length() - 1); + ByteString result; + if (!selfType) { + result = unescapeBytes(escaped); + } else { + result = unescapeBytesSelfType(escaped, fieldName); + } + nextToken(); + return result; + } catch (InvalidEscapeSequence e) { + throw parseException(e.getMessage()); + } catch (IllegalArgumentException e) { + throw parseException("INVALID base58 String, " + e.getMessage()); + } + } + + /** + * Returns a {@link ParseException} with the current line and column numbers in the description, + * suitable for throwing. + */ + public ParseException parseException(String description) { + // Note: People generally prefer one-based line and column numbers. + return new ParseException((line + 1) + ":" + (column + 1) + ": " + description); + } + + /** + * Returns a {@link ParseException} with the line and column numbers of the previous token in + * the description, suitable for throwing. + */ + public ParseException parseExceptionPreviousToken(String description) { + // Note: People generally prefer one-based line and column numbers. + return new ParseException((previousLine + 1) + ":" + (previousColumn + 1) + ": " + + description); + } + + /** + * Constructs an appropriate {@link ParseException} for the given {@code NumberFormatException} + * when trying to parse an integer. + */ + private ParseException integerParseException(NumberFormatException e) { + return parseException("Couldn't parse integer: " + e.getMessage()); + } + + /** + * Constructs an appropriate {@link ParseException} for the given {@code NumberFormatException} + * when trying to parse a float or double. + */ + private ParseException floatParseException(NumberFormatException e) { + return parseException("Couldn't parse number: " + e.getMessage()); + } + } + + /** + * Thrown when parsing an invalid text format message. + */ + public static class ParseException extends IOException { + + private static final long serialVersionUID = 1L; + + public ParseException(String message) { + super(message); + } + } + + /** + * Thrown by {@link JsonFormat#unescapeBytes} and {@link JsonFormat#unescapeText} when an invalid + * escape sequence is seen. + */ + static class InvalidEscapeSequence extends IOException { + + private static final long serialVersionUID = 1L; + + public InvalidEscapeSequence(String description) { + super(description); + } + } +} diff --git a/tools/toolkit/src/main/resources/fork.conf b/tools/toolkit/src/main/resources/fork.conf new file mode 100644 index 00000000..f99896e3 --- /dev/null +++ b/tools/toolkit/src/main/resources/fork.conf @@ -0,0 +1,58 @@ +witnesses = [ + { + address = "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" + url = "http://meme5.com" + voteCount = 100000036 + }, + { + address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" + voteCount = 100000035 + }, + { + address = "TKmyxLsRR2FWMVEHaQA2pZh1xB7oXPXzG1" + } +] + +accounts = [ + { + address = "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" + accountName = "Meme" + balance = 99000000000000000 + }, + { + address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" + accountType = "Normal" + balance = 99000000000000000 + }, + { + address = "TLLM21wteSPs4hKjbxgmH1L6poyMjeTbHm" + owner = "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" + }, + { + address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" + trc10Id = "1000001" + trc10Balance = 100000000 + }, + { + address = "TKmyxLsRR2FWMVEHaQA2pZh1xB7oXPXzG1" + } +] + +trc20Contracts = [ + { + contractAddress = "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t" + balancesSlotPosition = 0 + address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" + balance = "98800000000000000" + }, + { + contractAddress = "TSSMHYeV2uE9qYH95DqyoCuNCzEL1NvU3S" + balancesSlotPosition = 0 + address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" + balance = "128745186062510400000000000" + } +] + +latestBlockHeaderTimestamp = 1747986162000 # MilliSecond. Used in init setup, setting to current timestamp reduces debug log output and speeds up initialization. This setting does not impact the underlying block production logic. +maintenanceTimeInterval = 21600000 +nextMaintenanceTime = 1747996162000 # MilliSecond. Set next maintenance time. It could be anytime. If you set an expired time, it will trigger maintenance logic immediately after node startup. diff --git a/tools/toolkit/src/main/resources/logback.xml b/tools/toolkit/src/main/resources/logback.xml index 6c415042..ecba0088 100644 --- a/tools/toolkit/src/main/resources/logback.xml +++ b/tools/toolkit/src/main/resources/logback.xml @@ -52,6 +52,7 @@ - + + diff --git a/tools/toolkit/src/main/resources/query.conf b/tools/toolkit/src/main/resources/query.conf new file mode 100644 index 00000000..a1df2aee --- /dev/null +++ b/tools/toolkit/src/main/resources/query.conf @@ -0,0 +1,20 @@ +vote = { + allWitnesses = true + witnessList = [ + "TPDa1KRmFpLA42WffiXBwEykef9exHNPKV", + "TXnqPcy9zWHYyCiQ7Kmvt3aCx5jm9Qnq3e", + "TCZvvbn4SCVyNhCAt1L8Kp1qk5rtMiKdBB", + "TKPepDpU2FfagEFRy4eaN2MTevWjpzDNmT", + "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3", + "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" + ] +} + +reward = [ + "TLyqzVGLV1srkB7dToTAEqgDSfPtXRJZYH", + "TJvaAeFb8Lykt9RQcVyyTFN2iDvGMuyD4M", + "TNq7jKkTfFEvGXeqqZR5nk6wFse31wXv8B", + "TWocEBj78sJusnVq7xdXx2MR4LQNTqjkc4", + "TDaQAfV7gZSPzx5FZ23kenPwLjr3AGxnmH", + "TXwrVcWA2BPRMs9gGz5DciT6iwdDKagbpB" +] diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java new file mode 100644 index 00000000..3eeb05fa --- /dev/null +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java @@ -0,0 +1,227 @@ +package org.tron.plugins; + +import static org.tron.plugins.DbFork.getActiveWitness; +import static org.tron.plugins.utils.Constant.ACCOUNTS_KEY; +import static org.tron.plugins.utils.Constant.ACCOUNT_ADDRESS; +import static org.tron.plugins.utils.Constant.ACCOUNT_ASSET; +import static org.tron.plugins.utils.Constant.ACCOUNT_BALANCE; +import static org.tron.plugins.utils.Constant.ACCOUNT_NAME; +import static org.tron.plugins.utils.Constant.ACCOUNT_OWNER; +import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; +import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_BALANCE; +import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_ID; +import static org.tron.plugins.utils.Constant.ACCOUNT_TYPE; +import static org.tron.plugins.utils.Constant.ACTIVE_WITNESSES; +import static org.tron.plugins.utils.Constant.ASSET_ISSUE_V2; +import static org.tron.plugins.utils.Constant.CONTRACT_STORE; +import static org.tron.plugins.utils.Constant.DYNAMIC_PROPERTY_STORE; +import static org.tron.plugins.utils.Constant.LATEST_BLOCK_HEADER_TIMESTAMP; +import static org.tron.plugins.utils.Constant.LATEST_BLOCK_TIMESTAMP; +import static org.tron.plugins.utils.Constant.MAINTENANCE_INTERVAL; +import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME; +import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME_INTERVAL; +import static org.tron.plugins.utils.Constant.NEXT_MAINTENANCE_TIME; +import static org.tron.plugins.utils.Constant.STORAGE_ROW_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_ADDRESS; +import static org.tron.plugins.utils.Constant.WITNESS_KEY; +import static org.tron.plugins.utils.Constant.WITNESS_SCHEDULE_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_URL; +import static org.tron.plugins.utils.Constant.WITNESS_VOTE; + +import com.google.common.primitives.Bytes; +import com.google.protobuf.ByteString; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Paths; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.Commons; +import org.tron.core.capsule.AccountCapsule; +import org.tron.core.capsule.WitnessCapsule; +import org.tron.plugins.utils.Constant; +import org.tron.plugins.utils.FileUtils; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DbTool; +import picocli.CommandLine; + +public class DbForkTest { + + private DBInterface witnessStore; + private DBInterface witnessScheduleStore; + private DBInterface accountStore; + private DBInterface dynamicPropertiesStore; + private DBInterface accountAssetStore; + private DBInterface assetIssueV2Store; + + @Rule + public final TemporaryFolder folder = new TemporaryFolder(); + private String dbPath; + + public void createDir() { + String srcDir = dbPath + File.separator + "database"; + FileUtils.createDirIfNotExists(Paths.get(srcDir, WITNESS_STORE).toString()); + FileUtils.createDirIfNotExists(Paths.get(srcDir, WITNESS_SCHEDULE_STORE).toString()); + FileUtils.createDirIfNotExists(Paths.get(srcDir, ACCOUNT_STORE).toString()); + FileUtils.createDirIfNotExists(Paths.get(srcDir, DYNAMIC_PROPERTY_STORE).toString()); + + FileUtils.createDirIfNotExists(Paths.get(srcDir, ACCOUNT_ASSET).toString()); + FileUtils.createDirIfNotExists(Paths.get(srcDir, ASSET_ISSUE_V2).toString()); + FileUtils.createDirIfNotExists(Paths.get(srcDir, CONTRACT_STORE).toString()); + FileUtils.createDirIfNotExists(Paths.get(srcDir, STORAGE_ROW_STORE).toString()); + + } + + public void init() throws IOException, RocksDBException { + + String srcDir = dbPath + File.separator + "database"; + witnessStore = DbTool.getDB(srcDir, Constant.WITNESS_STORE); + witnessScheduleStore = DbTool.getDB(srcDir, Constant.WITNESS_SCHEDULE_STORE); + accountStore = DbTool.getDB(srcDir, Constant.ACCOUNT_STORE); + dynamicPropertiesStore = DbTool.getDB(srcDir, Constant.DYNAMIC_PROPERTY_STORE); + accountAssetStore = DbTool.getDB(srcDir, Constant.ACCOUNT_ASSET); + assetIssueV2Store = DbTool.getDB(srcDir, Constant.ASSET_ISSUE_V2); + } + + public void close() { + DbTool.close(); + } + + @Test + public void testDbFork() throws IOException, RocksDBException { + dbPath = folder.newFolder().toString(); + String forkPath = getConfig("fork.conf"); + createDir(); + + String[] args = new String[]{"-d", + dbPath, "-c", + forkPath}; + CommandLine cli = new CommandLine(new DbFork()); + Assert.assertEquals(0, cli.execute(args)); + + init(); + Config forkConfig; + File file = Paths.get(forkPath).toFile(); + if (file.exists() && file.isFile()) { + forkConfig = ConfigFactory.parseFile(Paths.get(forkPath).toFile()); + } else { + throw new IOException("Fork config file [" + forkPath + "] not exist!"); + } + + if (forkConfig.hasPath(WITNESS_KEY)) { + List witnesses = forkConfig.getConfigList(WITNESS_KEY); + if (witnesses.isEmpty()) { + System.out.println("no witness listed in the config."); + } + witnesses = witnesses.stream() + .filter(c -> c.hasPath(WITNESS_ADDRESS)) + .collect(Collectors.toList()); + if (witnesses.isEmpty()) { + System.out.println("no witness listed in the config."); + } + + List witnessAddresses = witnesses.stream().map( + w -> { + ByteString address = ByteString.copyFrom( + Commons.decodeFromBase58Check(w.getString(WITNESS_ADDRESS))); + return address; + } + ).collect(Collectors.toList()); + Assert.assertArrayEquals(getActiveWitness(witnessAddresses), + witnessScheduleStore.get(ACTIVE_WITNESSES)); + + witnesses.stream().forEach( + w -> { + WitnessCapsule witnessCapsule = new WitnessCapsule(witnessStore.get( + Commons.decodeFromBase58Check(w.getString(WITNESS_ADDRESS)))); + if (w.hasPath(WITNESS_VOTE)) { + Assert.assertEquals(w.getLong(WITNESS_VOTE), witnessCapsule.getVoteCount()); + } + if (w.hasPath(WITNESS_URL)) { + Assert.assertEquals(w.getString(WITNESS_URL), witnessCapsule.getUrl()); + } + } + ); + } + + if (forkConfig.hasPath(ACCOUNTS_KEY)) { + List accounts = forkConfig.getConfigList(ACCOUNTS_KEY); + if (accounts.isEmpty()) { + System.out.println("no account listed in the config."); + } + accounts = accounts.stream() + .filter(c -> c.hasPath(ACCOUNT_ADDRESS)) + .collect(Collectors.toList()); + if (accounts.isEmpty()) { + System.out.println("no account listed in the config."); + } + accounts.stream().forEach( + a -> { + byte[] address = Commons.decodeFromBase58Check(a.getString(ACCOUNT_ADDRESS)); + AccountCapsule account = new AccountCapsule(accountStore.get(address)); + Assert.assertNotNull(account); + if (a.hasPath(ACCOUNT_BALANCE)) { + Assert.assertEquals(a.getLong(ACCOUNT_BALANCE), account.getBalance()); + } + if (a.hasPath(ACCOUNT_NAME)) { + Assert.assertArrayEquals(ByteArray.fromString(a.getString(ACCOUNT_NAME)), + account.getAccountName().toByteArray()); + } + if (a.hasPath(ACCOUNT_TYPE)) { + Assert.assertEquals(a.getString(ACCOUNT_TYPE), account.getType().toString()); + } + if (a.hasPath(ACCOUNT_OWNER)) { + Assert.assertArrayEquals(Commons.decodeFromBase58Check(a.getString(ACCOUNT_OWNER)), + account.getPermissionById(0).getKeys(0).getAddress().toByteArray()); + } + if (a.hasPath(ACCOUNT_TRC10_ID) && a.hasPath(ACCOUNT_TRC10_BALANCE) + && a.getLong(ACCOUNT_TRC10_BALANCE) > 0) { + String trc10Id = a.getString(ACCOUNT_TRC10_ID); + if (assetIssueV2Store.get(ByteArray.fromString(trc10Id)) != null) { + if (account.getAssetOptimized()) { + byte[] k = Bytes.concat(address, ByteArray.fromString(trc10Id)); + byte[] value = accountAssetStore.get(k); + Assert.assertEquals(a.getLong(ACCOUNT_TRC10_BALANCE), ByteArray.toLong(value)); + } else { + long value = account.getAssetMapV2().get(trc10Id); + Assert.assertEquals(a.getLong(ACCOUNT_TRC10_BALANCE), value); + } + } + } + }); + } + + if (forkConfig.hasPath(LATEST_BLOCK_TIMESTAMP)) { + long latestBlockHeaderTimestamp = forkConfig.getLong(LATEST_BLOCK_TIMESTAMP); + Assert.assertEquals(latestBlockHeaderTimestamp, + ByteArray.toLong(dynamicPropertiesStore.get(LATEST_BLOCK_HEADER_TIMESTAMP))); + } + + if (forkConfig.hasPath(MAINTENANCE_INTERVAL)) { + long maintenanceTimeInterval = forkConfig.getLong(MAINTENANCE_INTERVAL); + Assert.assertEquals(maintenanceTimeInterval, + ByteArray.toLong(dynamicPropertiesStore.get(MAINTENANCE_TIME_INTERVAL))); + } + + if (forkConfig.hasPath(NEXT_MAINTENANCE_TIME)) { + long nextMaintenanceTime = forkConfig.getLong(NEXT_MAINTENANCE_TIME); + Assert.assertEquals(nextMaintenanceTime, + ByteArray.toLong(dynamicPropertiesStore.get(MAINTENANCE_TIME))); + } + close(); + } + + private String getConfig(String config) { + URL path = DbForkTest.class.getClassLoader().getResource(config); + return path == null ? null : path.getPath(); + } +} diff --git a/tools/toolkit/src/test/resources/fork.conf b/tools/toolkit/src/test/resources/fork.conf new file mode 100644 index 00000000..602eb4d6 --- /dev/null +++ b/tools/toolkit/src/test/resources/fork.conf @@ -0,0 +1,58 @@ +witnesses = [ + { + address = "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" + url = "http://meme5.com" + voteCount = 100000036 + }, + { + address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" + voteCount = 100000035 + }, + { + address = "TKmyxLsRR2FWMVEHaQA2pZh1xB7oXPXzG1" + } +] + +accounts = [ + { + address = "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" + accountName = "Meme" + balance = 99000000000000000 + }, + { + address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" + accountType = "Normal" + balance = 99000000000000000 + }, + { + address = "TLLM21wteSPs4hKjbxgmH1L6poyMjeTbHm" + owner = "TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W" + }, + { + address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" + trc10Id = "1000001" + trc10Balance = 100000000 + }, + { + address = "TKmyxLsRR2FWMVEHaQA2pZh1xB7oXPXzG1" + } +] + +trc20Contracts = [ + { + contractAddress = "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t" + balancesSlotPosition = 0 + address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" + balance = "98800000000000000" + }, + { + contractAddress = "TSSMHYeV2uE9qYH95DqyoCuNCzEL1NvU3S" + balancesSlotPosition = 0 + address = "TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3" + balance = "128745186062510400000000000" + } +] + +latestBlockHeaderTimestamp = 1739269661000 +maintenanceTimeInterval = 21600000 +nextMaintenanceTime = 1735628894000 From 520621b3ac2831ccc998e5b69cc75b6ef47192d2 Mon Sep 17 00:00:00 2001 From: federico Date: Mon, 26 Jan 2026 16:20:47 +0800 Subject: [PATCH 04/11] rm the useless code --- tools/stress_test/src/main/java/org/tron/BroadcastTx.java | 3 --- tools/stress_test/src/main/java/org/tron/CollectAddress.java | 3 --- .../toolkit/src/main/java/common/org/tron/plugins/DbLite.java | 3 --- tools/toolkit/src/main/resources/logback.xml | 1 - tools/toolkit/src/test/resources/config-duplicate.conf | 2 -- tools/toolkit/src/test/resources/config.conf | 2 -- 6 files changed, 14 deletions(-) diff --git a/tools/stress_test/src/main/java/org/tron/BroadcastTx.java b/tools/stress_test/src/main/java/org/tron/BroadcastTx.java index b543acc5..c7c91bb6 100644 --- a/tools/stress_test/src/main/java/org/tron/BroadcastTx.java +++ b/tools/stress_test/src/main/java/org/tron/BroadcastTx.java @@ -7,7 +7,6 @@ import java.nio.file.Paths; import java.util.concurrent.Callable; import lombok.extern.slf4j.Slf4j; -import org.checkerframework.checker.units.qual.A; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; @@ -15,7 +14,6 @@ import org.tron.core.config.args.Args; import org.tron.core.net.TronNetDelegate; import org.tron.core.net.TronNetService; -import org.tron.core.services.RpcApiService; import org.tron.trident.core.ApiWrapper; import org.tron.trident.core.exceptions.IllegalException; import org.tron.trident.proto.Chain.Block; @@ -118,7 +116,6 @@ public Integer call() throws IOException, InterruptedException, IllegalException context = new TronApplicationContext(DefaultConfig.class); app = ApplicationFactory.create(context); -// app.addService(context.getBean(RpcApiService.class)); app.startup(); String url = String.format("%s:%d", "127.0.0.1", diff --git a/tools/stress_test/src/main/java/org/tron/CollectAddress.java b/tools/stress_test/src/main/java/org/tron/CollectAddress.java index cbf47f75..1db8019c 100644 --- a/tools/stress_test/src/main/java/org/tron/CollectAddress.java +++ b/tools/stress_test/src/main/java/org/tron/CollectAddress.java @@ -1,7 +1,5 @@ package org.tron; -//import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; - import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import com.typesafe.config.Config; @@ -130,7 +128,6 @@ private Set getAddressListFromDB(String dbPath, int totalNumber) throws IOException, RocksDBException { Set addressList = new HashSet<>(); String srcDir = dbPath + File.separator + "database"; -// DBInterface accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); DBInterface accountStore = DbTool.getDB(srcDir, "account"); DBIterator iterator = accountStore.iterator(); for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java index 3f8a6cb5..5ce2a445 100644 --- a/tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java @@ -721,6 +721,3 @@ public long getSnapshotMaxNum() { } } } - - - diff --git a/tools/toolkit/src/main/resources/logback.xml b/tools/toolkit/src/main/resources/logback.xml index ecba0088..555edd23 100644 --- a/tools/toolkit/src/main/resources/logback.xml +++ b/tools/toolkit/src/main/resources/logback.xml @@ -55,4 +55,3 @@ - diff --git a/tools/toolkit/src/test/resources/config-duplicate.conf b/tools/toolkit/src/test/resources/config-duplicate.conf index f2eb7fbf..05b7b9f4 100644 --- a/tools/toolkit/src/test/resources/config-duplicate.conf +++ b/tools/toolkit/src/test/resources/config-duplicate.conf @@ -17,5 +17,3 @@ storage { }, ] } - - diff --git a/tools/toolkit/src/test/resources/config.conf b/tools/toolkit/src/test/resources/config.conf index 77d15d52..df7b703a 100644 --- a/tools/toolkit/src/test/resources/config.conf +++ b/tools/toolkit/src/test/resources/config.conf @@ -11,5 +11,3 @@ storage { }, ] } - - From 514c4088c1037b96c5dae4ee0610cb2d6b7a9cf6 Mon Sep 17 00:00:00 2001 From: federico Date: Mon, 26 Jan 2026 19:33:14 +0800 Subject: [PATCH 05/11] fix the format --- tools/stress_test/build.gradle | 8 ++++---- tools/toolkit/build.gradle | 11 ++++------- .../java/x86/org/tron/plugins/ArchiveManifest.java | 2 +- .../src/main/java/x86/org/tron/plugins/DbArchive.java | 3 +-- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/tools/stress_test/build.gradle b/tools/stress_test/build.gradle index 8c2108aa..e7616208 100644 --- a/tools/stress_test/build.gradle +++ b/tools/stress_test/build.gradle @@ -47,10 +47,10 @@ dependencies { // implementation 'ch.qos.logback:logback-classic:1.2.6' implementation group: 'info.picocli', name: 'picocli', version: '4.6.3' - compileOnly 'org.projectlombok:lombok:1.18.12' - annotationProcessor 'org.projectlombok:lombok:1.18.12' - testCompileOnly 'org.projectlombok:lombok:1.18.12' - testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' + compileOnly 'org.projectlombok:lombok:1.18.34' + annotationProcessor 'org.projectlombok:lombok:1.18.34' + testCompileOnly 'org.projectlombok:lombok:1.18.34' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.34' implementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.20') { exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' diff --git a/tools/toolkit/build.gradle b/tools/toolkit/build.gradle index b19e1048..6daad931 100644 --- a/tools/toolkit/build.gradle +++ b/tools/toolkit/build.gradle @@ -55,9 +55,6 @@ dependencies { implementation group: 'com.typesafe', name: 'config', version: '1.3.2' implementation group: 'me.tongfei', name: 'progressbar', version: '0.9.3' implementation group: 'org.bouncycastle', name: 'bcprov-jdk18on', version: '1.79' - implementation group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10' - implementation 'io.github.tronprotocol:leveldbjni-all:1.18.2' - implementation 'io.github.tronprotocol:leveldb:1.18.2' implementation('com.github.federico2014.java-tron:chainbase:GreatVoyage-v4.8.20') { exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' @@ -66,10 +63,10 @@ dependencies { implementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') implementation group: 'info.picocli', name: 'picocli', version: '4.6.3' - compileOnly 'org.projectlombok:lombok:1.18.12' - annotationProcessor 'org.projectlombok:lombok:1.18.12' - testCompileOnly 'org.projectlombok:lombok:1.18.12' - testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' + compileOnly 'org.projectlombok:lombok:1.18.34' + annotationProcessor 'org.projectlombok:lombok:1.18.34' + testCompileOnly 'org.projectlombok:lombok:1.18.34' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.34' implementation 'com.alibaba:fastjson:1.2.83' implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25' diff --git a/tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java b/tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java index 1d7a9102..115144be 100644 --- a/tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java +++ b/tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java @@ -269,4 +269,4 @@ public static class Args { @Option(names = {"-h", "--help"}, help = true) private boolean help; } -} \ No newline at end of file +} diff --git a/tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java b/tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java index 15bb281b..1088468f 100644 --- a/tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java +++ b/tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java @@ -195,5 +195,4 @@ public void doArchive() { } } } - -} \ No newline at end of file +} From 9dcbf20eb32abcb238d606ba873b50c192d68923 Mon Sep 17 00:00:00 2001 From: federico Date: Mon, 26 Jan 2026 16:53:59 +0800 Subject: [PATCH 06/11] fix the format --- tools/toolkit/src/main/resources/fork.conf | 78 ++++++++++++++++++++- tools/toolkit/src/main/resources/query.conf | 11 +++ 2 files changed, 86 insertions(+), 3 deletions(-) diff --git a/tools/toolkit/src/main/resources/fork.conf b/tools/toolkit/src/main/resources/fork.conf index f99896e3..25c12874 100644 --- a/tools/toolkit/src/main/resources/fork.conf +++ b/tools/toolkit/src/main/resources/fork.conf @@ -53,6 +53,78 @@ trc20Contracts = [ } ] -latestBlockHeaderTimestamp = 1747986162000 # MilliSecond. Used in init setup, setting to current timestamp reduces debug log output and speeds up initialization. This setting does not impact the underlying block production logic. -maintenanceTimeInterval = 21600000 -nextMaintenanceTime = 1747996162000 # MilliSecond. Set next maintenance time. It could be anytime. If you set an expired time, it will trigger maintenance logic immediately after node startup. +chainParameters = { + latestBlockHeaderTimestamp = 1747986162000 # MilliSecond. Used in init setup, setting to current timestamp reduces debug log output and speeds up initialization. This setting does not impact the underlying block production logic. + maintenanceTimeInterval = 11100000 + nextMaintenanceTime = 1747996162000 # MilliSecond. Set next maintenance time. It could be anytime. If you set an expired time, it will trigger maintenance logic immediately after node startup. + // accountUpgradeCost = 9999000000 + // createAccountFee = 550000 + // transactionFee = 1000 + // assetIssueFee = 1024000000 + // witnessPayPerBlock = 8000000 + // witnessStandbyAllowance = 115200000000 + // createNewAccountFeeInSystemContract = 1000000 + // createNewAccountBandwidthRate = 1 + // allowCreationOfContracts = 1 + // removeThePowerOfTheGr = -1 + // energyFee = 223 + // exchangeCreateFee = 1024000000 + maxCpuTimeOfOneTx = 300 + // allowUpdateAccountName = 0 + // allowSameTokenName = 1 + // allowDelegateResource = 1 + // totalEnergyLimit = 180000000000 + // allowTvmTransferTrc10 = 1 + // totalEnergyCurrentLimit = 180000000000 + // allowMultiSign = 1 + // allowAdaptiveEnergy = 0 + // totalEnergyTargetLimit = 12500000 + // totalEnergyAverageUsage = 0 + // updateAccountPermissionFee = 100000000 + // multiSignFee = 1000000 + // allowAccountStateRoot = 0 + // allowProtoFilterNum = 0 + // allowTvmConstantinople = 1 + // allowTvmSolidity059 = 1 + // allowTvmIstanbul = 1 + // allowShieldedTRC20Transaction = 1 + // forbidTransferToContract = 0 + // adaptiveResourceLimitTargetRatio = 10 + // adaptiveResourceLimitMultiplier = 1000 + // changeDelegation = 1 + // witness127PayPerBlock = 128000000 + // allowMarketTransaction = 0 + // marketSellFee = 0 + // marketCancelFee = 0 + // allowPBFT = 0 + // allowTransactionFeePool = 0 + // maxFeeLimit = 15000000000 + // allowOptimizeBlackHole = 1 + // allowNewResourceModel = 0 + // allowTvmFreeze = 0 + // allowTvmVote = 1 + // allowTvmLondon = 1 + // allowTvmCompatibleEvm = 0 + // allowAccountAssetOptimization = 0 + // freeNetLimit = 600 + // totalNetLimit = 43200000000 + // allowHigherLimitForMaxCpuTimeOfOneTx = 1 + // allowAssetOptimization = 1 + // allowNewReward = 1 + // memoFee = 1000000 + // allowDelegateOptimization = 1 + // unfreezeDelayDays = 14 + // allowOptimizedReturnValueOfChainId = 1 + // allowDynamicEnergy = 1 + // dynamicEnergyThreshold = 5000000000 + // dynamicEnergyIncreaseFactor = 3400 + // allowTvmShangHai = 1 + // allowCancelAllUnfreezeV2 = 1 + // maxDelegateLockPeriod = 864000 + // allowEnergyAdjustment = 1 + // maxCreateAccountTxSize = 1000 + // allowStrictMath = 1 + // consensusLogicOptimization = 1 + // allowTvmCancun = 1 + // allowTvmBlob = 1 +} diff --git a/tools/toolkit/src/main/resources/query.conf b/tools/toolkit/src/main/resources/query.conf index a1df2aee..17ba2f5d 100644 --- a/tools/toolkit/src/main/resources/query.conf +++ b/tools/toolkit/src/main/resources/query.conf @@ -18,3 +18,14 @@ reward = [ "TDaQAfV7gZSPzx5FZ23kenPwLjr3AGxnmH", "TXwrVcWA2BPRMs9gGz5DciT6iwdDKagbpB" ] + +// query the voters of the specified witnesses satisfying the threshold. +voters = { + allWitnesses = false + allActiveWitnesses = true + witnessList = [ + "TMEQ4hu7DtLTHPn6qPsYktk1QGdTbAyTEK", + "THKJYuUmMKKARNf7s2VT51g5uPY6KEqnat" + ] + threshold = 0 +} From eb768b2d9d23747842811066eba41e8f72657778 Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 27 Jan 2026 16:04:43 +0800 Subject: [PATCH 07/11] support arm64 rocksdb --- .../main/java/common/org/tron/plugins/utils/db/DbTool.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java index 127b8f97..d9962394 100644 --- a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java @@ -9,6 +9,7 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksDBException; +import org.tron.common.arch.Arch; import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.FileUtils; @@ -164,7 +165,8 @@ public static void close() { private static DbType getDbType(String sourceDir, String dbName) { String engineFile = Paths.get(sourceDir, dbName, ENGINE_FILE).toString(); if (!new File(engineFile).exists()) { - return DbType.LevelDB; + // For arm64, only support RocksDB + return Arch.isArm64() ? DbType.RocksDB : DbType.LevelDB; } String engine = FileUtils.readProperty(engineFile, KEY_ENGINE); if (engine.equalsIgnoreCase(ROCKSDB)) { From ef49e8bb0ad46382b3dd60db6929b787c85313b2 Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 27 Jan 2026 16:44:36 +0800 Subject: [PATCH 08/11] modify build.gradle --- tools/toolkit/build.gradle | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/toolkit/build.gradle b/tools/toolkit/build.gradle index 6daad931..642ec590 100644 --- a/tools/toolkit/build.gradle +++ b/tools/toolkit/build.gradle @@ -20,6 +20,13 @@ repositories { maven { url 'https://jitpack.io' } } +// Force RocksDB version based on architecture (same as java-tron/platform module) +configurations.all { + resolutionStrategy { + force "org.rocksdb:rocksdbjni:${rootProject.archInfo.requires.RocksdbVersion}" + } +} + tasks.named('jar') { enabled = false dependsOn shadowJar @@ -124,3 +131,20 @@ task checkstyleToolkitTest(type: Checkstyle) { } check.dependsOn checkstyleToolkitMain, checkstyleToolkitTest + +test { + testLogging { + exceptionFormat = 'full' + } + + if (rootProject.archInfo.isArm64) { + exclude 'org/tron/plugins/leveldb/**' + filter { + excludeTestsMatching '*.*leveldb*' + excludeTestsMatching '*.*Leveldb*' + excludeTestsMatching '*.*LevelDB*' + excludeTestsMatching '*.*LevelDb*' + excludeTestsMatching '*.*Archive*' + } + } +} From 785ff61ee41058cd41a153b716c21a2fbe0930ad Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 27 Jan 2026 17:48:15 +0800 Subject: [PATCH 09/11] build.gradle arm64 support --- tools/toolkit/build.gradle | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tools/toolkit/build.gradle b/tools/toolkit/build.gradle index 642ec590..c0f61cc2 100644 --- a/tools/toolkit/build.gradle +++ b/tools/toolkit/build.gradle @@ -51,11 +51,6 @@ sourceSets { dependencies { testImplementation 'junit:junit:4.13.2' - testImplementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.20') { - exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' - exclude group: "pull-parser", module: "pull-parser" - } - testImplementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') implementation group: 'org.springframework', name: 'spring-context', version: "${springVersion}" implementation group: 'info.picocli', name: 'picocli', version: '4.6.3' @@ -63,13 +58,6 @@ dependencies { implementation group: 'me.tongfei', name: 'progressbar', version: '0.9.3' implementation group: 'org.bouncycastle', name: 'bcprov-jdk18on', version: '1.79' - implementation('com.github.federico2014.java-tron:chainbase:GreatVoyage-v4.8.20') { - exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' - exclude group: "pull-parser", module: "pull-parser" - } - implementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') - implementation group: 'info.picocli', name: 'picocli', version: '4.6.3' - compileOnly 'org.projectlombok:lombok:1.18.34' annotationProcessor 'org.projectlombok:lombok:1.18.34' testCompileOnly 'org.projectlombok:lombok:1.18.34' @@ -80,11 +68,29 @@ dependencies { implementation group: 'org.slf4j', name: 'jcl-over-slf4j', version: '1.7.25' implementation "org.apache.commons:commons-lang3:3.4" implementation "org.apache.commons:commons-collections4:4.1" + implementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') if (rootProject.archInfo.isArm64) { - testRuntimeOnly group: 'org.fusesource.hawtjni', name: 'hawtjni-runtime', version: '1.18' // for test - implementation('com.github.federico2014.java-tron:platform:GreatVoyage-v4.8.20') + // ARM64 + testImplementation('com.github.warku123.java-tron:framework:GreatVoyage-v4.8.0-arm') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + } + implementation('com.github.warku123.java-tron:chainbase:GreatVoyage-v4.8.0-arm') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + } + implementation 'com.github.warku123.java-tron:platform:GreatVoyage-v4.8.0-arm' } else { + // x86 + testImplementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.20') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + } + implementation('com.github.federico2014.java-tron:chainbase:GreatVoyage-v4.8.20') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + } implementation('com.github.federico2014.java-tron:platform:GreatVoyage-v4.8.20') { exclude(group: 'org.fusesource.leveldbjni', module: 'leveldbjni-all') exclude(group: 'io.github.tronprotocol', module: 'zksnark-java-sdk') From 9a632a2c33a88515f4914753cee5a6fd623f2e93 Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 27 Jan 2026 17:59:00 +0800 Subject: [PATCH 10/11] update build tag --- tools/toolkit/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/toolkit/build.gradle b/tools/toolkit/build.gradle index c0f61cc2..ea9dc1e4 100644 --- a/tools/toolkit/build.gradle +++ b/tools/toolkit/build.gradle @@ -72,15 +72,15 @@ dependencies { if (rootProject.archInfo.isArm64) { // ARM64 - testImplementation('com.github.warku123.java-tron:framework:GreatVoyage-v4.8.0-arm') { + testImplementation('com.github.warku123.java-tron:framework:GreatVoyage-v4.8.1-arm') { exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' exclude group: "pull-parser", module: "pull-parser" } - implementation('com.github.warku123.java-tron:chainbase:GreatVoyage-v4.8.0-arm') { + implementation('com.github.warku123.java-tron:chainbase:GreatVoyage-v4.8.1-arm') { exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' exclude group: "pull-parser", module: "pull-parser" } - implementation 'com.github.warku123.java-tron:platform:GreatVoyage-v4.8.0-arm' + implementation 'com.github.warku123.java-tron:platform:GreatVoyage-v4.8.1-arm' } else { // x86 testImplementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.20') { From 8e4b7a706adba2447d86689c45effd2404b3eba1 Mon Sep 17 00:00:00 2001 From: federico Date: Wed, 28 Jan 2026 16:10:36 +0800 Subject: [PATCH 11/11] update the gradle --- tools/stress_test/build.gradle | 17 +++-- tools/toolkit/build.gradle | 6 +- tools/toolkit/src/main/resources/fork.conf | 78 +-------------------- tools/toolkit/src/main/resources/query.conf | 11 --- 4 files changed, 18 insertions(+), 94 deletions(-) diff --git a/tools/stress_test/build.gradle b/tools/stress_test/build.gradle index e7616208..0c8654a2 100644 --- a/tools/stress_test/build.gradle +++ b/tools/stress_test/build.gradle @@ -44,7 +44,6 @@ dependencies { implementation "io.netty:netty-all:4.1.100.Final" implementation("io.github.317787106:trident:0.9.2.5") implementation group: 'com.typesafe', name: 'config', version: '1.3.2' -// implementation 'ch.qos.logback:logback-classic:1.2.6' implementation group: 'info.picocli', name: 'picocli', version: '4.6.3' compileOnly 'org.projectlombok:lombok:1.18.34' @@ -52,10 +51,18 @@ dependencies { testCompileOnly 'org.projectlombok:lombok:1.18.34' testAnnotationProcessor 'org.projectlombok:lombok:1.18.34' - implementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.20') { - exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' - exclude group: "pull-parser", module: "pull-parser" - exclude group: "ch.qos.logback", module: "logback-classic" + if (rootProject.archInfo.isArm64) { + implementation('com.github.warku123.java-tron:framework:GreatVoyage-v4.8.1-arm') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + exclude group: "ch.qos.logback", module: "logback-classic" + } + } else { + implementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.1-x86_64') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + exclude group: "ch.qos.logback", module: "logback-classic" + } } implementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') implementation(project(":toolkit")) { diff --git a/tools/toolkit/build.gradle b/tools/toolkit/build.gradle index ea9dc1e4..4ba8b768 100644 --- a/tools/toolkit/build.gradle +++ b/tools/toolkit/build.gradle @@ -83,15 +83,15 @@ dependencies { implementation 'com.github.warku123.java-tron:platform:GreatVoyage-v4.8.1-arm' } else { // x86 - testImplementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.20') { + testImplementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.1-x86_64') { exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' exclude group: "pull-parser", module: "pull-parser" } - implementation('com.github.federico2014.java-tron:chainbase:GreatVoyage-v4.8.20') { + implementation('com.github.federico2014.java-tron:chainbase:GreatVoyage-v4.8.1-x86_64') { exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' exclude group: "pull-parser", module: "pull-parser" } - implementation('com.github.federico2014.java-tron:platform:GreatVoyage-v4.8.20') { + implementation('com.github.federico2014.java-tron:platform:GreatVoyage-v4.8.1-x86_64') { exclude(group: 'org.fusesource.leveldbjni', module: 'leveldbjni-all') exclude(group: 'io.github.tronprotocol', module: 'zksnark-java-sdk') exclude(group: 'commons-io', module: 'commons-io') diff --git a/tools/toolkit/src/main/resources/fork.conf b/tools/toolkit/src/main/resources/fork.conf index 25c12874..f99896e3 100644 --- a/tools/toolkit/src/main/resources/fork.conf +++ b/tools/toolkit/src/main/resources/fork.conf @@ -53,78 +53,6 @@ trc20Contracts = [ } ] -chainParameters = { - latestBlockHeaderTimestamp = 1747986162000 # MilliSecond. Used in init setup, setting to current timestamp reduces debug log output and speeds up initialization. This setting does not impact the underlying block production logic. - maintenanceTimeInterval = 11100000 - nextMaintenanceTime = 1747996162000 # MilliSecond. Set next maintenance time. It could be anytime. If you set an expired time, it will trigger maintenance logic immediately after node startup. - // accountUpgradeCost = 9999000000 - // createAccountFee = 550000 - // transactionFee = 1000 - // assetIssueFee = 1024000000 - // witnessPayPerBlock = 8000000 - // witnessStandbyAllowance = 115200000000 - // createNewAccountFeeInSystemContract = 1000000 - // createNewAccountBandwidthRate = 1 - // allowCreationOfContracts = 1 - // removeThePowerOfTheGr = -1 - // energyFee = 223 - // exchangeCreateFee = 1024000000 - maxCpuTimeOfOneTx = 300 - // allowUpdateAccountName = 0 - // allowSameTokenName = 1 - // allowDelegateResource = 1 - // totalEnergyLimit = 180000000000 - // allowTvmTransferTrc10 = 1 - // totalEnergyCurrentLimit = 180000000000 - // allowMultiSign = 1 - // allowAdaptiveEnergy = 0 - // totalEnergyTargetLimit = 12500000 - // totalEnergyAverageUsage = 0 - // updateAccountPermissionFee = 100000000 - // multiSignFee = 1000000 - // allowAccountStateRoot = 0 - // allowProtoFilterNum = 0 - // allowTvmConstantinople = 1 - // allowTvmSolidity059 = 1 - // allowTvmIstanbul = 1 - // allowShieldedTRC20Transaction = 1 - // forbidTransferToContract = 0 - // adaptiveResourceLimitTargetRatio = 10 - // adaptiveResourceLimitMultiplier = 1000 - // changeDelegation = 1 - // witness127PayPerBlock = 128000000 - // allowMarketTransaction = 0 - // marketSellFee = 0 - // marketCancelFee = 0 - // allowPBFT = 0 - // allowTransactionFeePool = 0 - // maxFeeLimit = 15000000000 - // allowOptimizeBlackHole = 1 - // allowNewResourceModel = 0 - // allowTvmFreeze = 0 - // allowTvmVote = 1 - // allowTvmLondon = 1 - // allowTvmCompatibleEvm = 0 - // allowAccountAssetOptimization = 0 - // freeNetLimit = 600 - // totalNetLimit = 43200000000 - // allowHigherLimitForMaxCpuTimeOfOneTx = 1 - // allowAssetOptimization = 1 - // allowNewReward = 1 - // memoFee = 1000000 - // allowDelegateOptimization = 1 - // unfreezeDelayDays = 14 - // allowOptimizedReturnValueOfChainId = 1 - // allowDynamicEnergy = 1 - // dynamicEnergyThreshold = 5000000000 - // dynamicEnergyIncreaseFactor = 3400 - // allowTvmShangHai = 1 - // allowCancelAllUnfreezeV2 = 1 - // maxDelegateLockPeriod = 864000 - // allowEnergyAdjustment = 1 - // maxCreateAccountTxSize = 1000 - // allowStrictMath = 1 - // consensusLogicOptimization = 1 - // allowTvmCancun = 1 - // allowTvmBlob = 1 -} +latestBlockHeaderTimestamp = 1747986162000 # MilliSecond. Used in init setup, setting to current timestamp reduces debug log output and speeds up initialization. This setting does not impact the underlying block production logic. +maintenanceTimeInterval = 21600000 +nextMaintenanceTime = 1747996162000 # MilliSecond. Set next maintenance time. It could be anytime. If you set an expired time, it will trigger maintenance logic immediately after node startup. diff --git a/tools/toolkit/src/main/resources/query.conf b/tools/toolkit/src/main/resources/query.conf index 17ba2f5d..a1df2aee 100644 --- a/tools/toolkit/src/main/resources/query.conf +++ b/tools/toolkit/src/main/resources/query.conf @@ -18,14 +18,3 @@ reward = [ "TDaQAfV7gZSPzx5FZ23kenPwLjr3AGxnmH", "TXwrVcWA2BPRMs9gGz5DciT6iwdDKagbpB" ] - -// query the voters of the specified witnesses satisfying the threshold. -voters = { - allWitnesses = false - allActiveWitnesses = true - witnessList = [ - "TMEQ4hu7DtLTHPn6qPsYktk1QGdTbAyTEK", - "THKJYuUmMKKARNf7s2VT51g5uPY6KEqnat" - ] - threshold = 0 -}