From 296a7848baed68af82e74c7ab17cfe8f6da97765 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Mon, 27 Apr 2026 20:50:37 +0800 Subject: [PATCH 1/2] feat(toolkit): add --exclude-historical-balance for lite snapshot split Default behavior is unchanged: balance-trace and account-trace stay in the lite snapshot as before, so default operators (historyBalanceLookup=off) see no difference. Opt-in via `--exclude-historical-balance=true` on `split -t snapshot` excludes the two trace stores from the snapshot for size-conscious operators. A loud warning is printed at split time noting that this loss is permanent for nodes that had historyBalanceLookup=true (merge cannot restore the feature) and that operators who need historical balance lookup on the resulting lite node must NOT enable this flag. `split -t history` and `merge` ignore the flag and continue using the legacy 5-db archive set, so merge logic stays untouched. Includes: - DbLite: new CLI option, helper method, runtime warning. - README: parameter documentation and worked example. - DbLiteTest: 3-arg testTools overload and packaging-contract assertion. - DbLiteExcludeHistoricalBalanceRocksDbTest: opt-in path coverage. close #6597 --- plugins/README.md | 6 +- .../java/common/org/tron/plugins/DbLite.java | 67 ++++++++++++++++++- .../java/org/tron/plugins/DbLiteTest.java | 28 +++++++- ...teExcludeHistoricalBalanceRocksDbTest.java | 13 ++++ 4 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteExcludeHistoricalBalanceRocksDbTest.java diff --git a/plugins/README.md b/plugins/README.md index b6540beef1a..c405223cab3 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -75,16 +75,20 @@ DB lite provides lite database, parameters are compatible with previous `LiteFul - `-fn | --fn-data-path`: The database path to be split or merged. - `-ds | --dataset-path`: When operation is `split`,`dataset-path` is the path that store the `snapshot` or `history`, when operation is `split`, `dataset-path` is the `history` data path. +- `--exclude-historical-balance`: Only used with `operate=split -t snapshot`, default: false. When set to true, `balance-trace` and `account-trace` are excluded from the lite snapshot. The flag has functional impact only when the source full node ran with `historyBalanceLookup=true` (off by default; most operators are unaffected). **WARNING:** for nodes that had `historyBalanceLookup=true`, this loss is permanent — a lite node booted from such a snapshot cannot answer historical balance lookups (`getBlockBalance` / `getAccountBalance`), and running `merge` afterwards will NOT restore the feature. If you need historical balance lookup on the resulting lite node, do **not** enable this flag. `split -t history` and `merge` ignore this flag. - `-h | --help`: Provide the help info. ### Examples: ```shell script # full command - java -jar Toolkit.jar db lite [-h] -ds= -fn= [-o=] [-t=] + java -jar Toolkit.jar db lite [-h] -ds= -fn= [-o=] [-t=] [--exclude-historical-balance] # examples #split and get a snapshot dataset java -jar Toolkit.jar db lite -o split -t snapshot --fn-data-path output-directory/database --dataset-path /tmp + #split and get a snapshot dataset without balance-trace / account-trace (smaller snapshot; + #historical balance lookup will be permanently unavailable on the resulting lite node) + java -jar Toolkit.jar db lite -o split -t snapshot --fn-data-path output-directory/database --dataset-path /tmp --exclude-historical-balance #split and get a history dataset java -jar Toolkit.jar db lite -o split -t history --fn-data-path output-directory/database --dataset-path /tmp #merge history dataset and snapshot dataset diff --git a/plugins/src/main/java/common/org/tron/plugins/DbLite.java b/plugins/src/main/java/common/org/tron/plugins/DbLite.java index 3f8a6cb58c8..2804cdc2800 100644 --- a/plugins/src/main/java/common/org/tron/plugins/DbLite.java +++ b/plugins/src/main/java/common/org/tron/plugins/DbLite.java @@ -20,6 +20,7 @@ import java.util.concurrent.Callable; import java.util.stream.Collectors; import java.util.stream.LongStream; +import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import me.tongfei.progressbar.ProgressBar; import org.rocksdb.RocksDBException; @@ -57,6 +58,8 @@ public class DbLite implements Callable { private static final String TRANSACTION_HISTORY_DB_NAME = "transactionHistoryStore"; private static final String PROPERTIES_DB_NAME = "properties"; private static final String TRANS_CACHE_DB_NAME = "trans-cache"; + private static final String BALANCE_TRACE_DB_NAME = "balance-trace"; + private static final String ACCOUNT_TRACE_DB_NAME = "account-trace"; private static final List archiveDbs = Arrays.asList( BLOCK_DB_NAME, @@ -65,6 +68,10 @@ public class DbLite implements Callable { TRANSACTION_RET_DB_NAME, TRANSACTION_HISTORY_DB_NAME); + private static final List traceDbs = Arrays.asList( + BALANCE_TRACE_DB_NAME, + ACCOUNT_TRACE_DB_NAME); + enum Operate { split, merge } enum Type { snapshot, history } @@ -105,8 +112,25 @@ enum Type { snapshot, history } private String datasetPath; @CommandLine.Option( - names = {"--help", "-h"}, + names = {"--exclude-historical-balance"}, + defaultValue = "false", + description = "only used with `operate=split -t snapshot`: when true, balance-trace " + + "and account-trace are excluded from the lite snapshot. " + + "Default: ${DEFAULT-VALUE} (legacy behavior; trace stores stay in the snapshot). " + + "This flag only has a functional impact when the source full node ran with " + + "`historyBalanceLookup=true` (off by default; most operators are unaffected). " + + "WARNING: when historyBalanceLookup was enabled, this loss is permanent: a lite " + + "node booted from such a snapshot cannot answer historical balance lookups " + + "(getBlockBalance / getAccountBalance), and running merge afterwards will NOT " + + "restore the feature. If you need to keep historyBalanceLookup working on the " + + "resulting lite node, do NOT enable this flag. `split -t history` and `merge` " + + "ignore this flag.", order = 5) + private boolean excludeHistoricalBalance; + + @CommandLine.Option( + names = {"--help", "-h"}, + order = 6) private boolean help; @@ -119,6 +143,7 @@ public Integer call() { try { switch (this.operate) { case split: + warnIfExcludingHistoricalBalance(); if (Type.snapshot == this.type) { generateSnapshot(fnDataPath, datasetPath); } else if (Type.history == type) { @@ -253,12 +278,50 @@ public void completeHistoryData(String historyDir, String liteDir) { spec.commandLine().getOut().format("Merge history finished, take %d s.", during).println(); } + /** + * Compute the directories to exclude from the lite snapshot. + *

+ * Default ({@code --exclude-historical-balance=false}): the legacy archive set + * (5 dbs); {@link #BALANCE_TRACE_DB_NAME} / {@link #ACCOUNT_TRACE_DB_NAME} + * stay with the snapshot as state-style stores. + *

+ * Opt-in ({@code --exclude-historical-balance=true}): the trace stores are + * additionally excluded, producing a smaller lite snapshot at the cost of + * dropping historical balance lookup support on the resulting lite node. + * Only {@code split -t snapshot} consults this. {@code split -t history} + * and {@code merge} always use the legacy archive set. + */ + private List snapshotExclusion() { + if (!excludeHistoricalBalance) { + return archiveDbs; + } + return Stream.concat(archiveDbs.stream(), traceDbs.stream()) + .collect(Collectors.toList()); + } + + private void warnIfExcludingHistoricalBalance() { + if (!excludeHistoricalBalance) { + return; + } + String msg = "WARNING: --exclude-historical-balance is enabled. balance-trace / account-trace " + + "will be excluded from the lite snapshot. This only matters when the source full " + + "node ran with historyBalanceLookup=true (off by default; most operators are " + + "unaffected). When that switch was enabled, this loss is permanent: lite nodes " + + "booted from this snapshot cannot answer historical balance lookups " + + "(getBlockBalance / getAccountBalance), and running merge afterwards will NOT " + + "restore the feature. If you need to keep historyBalanceLookup working on the " + + "resulting lite node, do NOT use this flag."; + logger.warn(msg); + spec.commandLine().getErr().println(msg); + } + private List getSnapshotDbs(String sourceDir) { List snapshotDbs = Lists.newArrayList(); File basePath = new File(sourceDir); + List excluded = snapshotExclusion(); Arrays.stream(Objects.requireNonNull(basePath.listFiles())) .filter(File::isDirectory) - .filter(dir -> !archiveDbs.contains(dir.getName())) + .filter(dir -> !excluded.contains(dir.getName())) .forEach(dir -> snapshotDbs.add(dir.getName())); return snapshotDbs; } diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java index 07bddc461a0..7702cb046cc 100644 --- a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java @@ -1,11 +1,14 @@ package org.tron.plugins; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.tron.common.utils.PublicMethod.getRandomPrivateKey; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.nio.file.Paths; import lombok.extern.slf4j.Slf4j; import org.junit.After; @@ -89,10 +92,19 @@ public void clear() { public void testTools(String dbType, int checkpointVersion) throws InterruptedException, IOException { - logger.info("dbType {}, checkpointVersion {}", dbType, checkpointVersion); + testTools(dbType, checkpointVersion, false); + } + + public void testTools(String dbType, int checkpointVersion, boolean excludeHistoricalBalance) + throws InterruptedException, IOException { + logger.info("dbType {}, checkpointVersion {}, excludeHistoricalBalance {}", + dbType, checkpointVersion, excludeHistoricalBalance); init(dbType); - final String[] argsForSnapshot = - new String[] {"-o", "split", "-t", "snapshot", "--fn-data-path", + final String[] argsForSnapshot = excludeHistoricalBalance + ? new String[] {"-o", "split", "-t", "snapshot", "--fn-data-path", + dbPath + File.separator + databaseDir, "--dataset-path", + dbPath, "--exclude-historical-balance"} + : new String[] {"-o", "split", "-t", "snapshot", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath}; final String[] argsForHistory = @@ -114,6 +126,16 @@ public void testTools(String dbType, int checkpointVersion) FileUtil.deleteDir(Paths.get(dbPath, databaseDir, "trans-cache").toFile()); // generate snapshot cli.execute(argsForSnapshot); + Path snapshotDir = Paths.get(dbPath, "snapshot"); + if (excludeHistoricalBalance) { + // when --exclude-historical-balance=true, the lite snapshot must not ship + // balance-trace / account-trace + assertFalse(snapshotDir.resolve("balance-trace").toFile().exists()); + assertFalse(snapshotDir.resolve("account-trace").toFile().exists()); + } else { + assertTrue(snapshotDir.resolve("balance-trace").toFile().exists()); + assertTrue(snapshotDir.resolve("account-trace").toFile().exists()); + } // start fullNode startApp(); // produce transactions diff --git a/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteExcludeHistoricalBalanceRocksDbTest.java b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteExcludeHistoricalBalanceRocksDbTest.java new file mode 100644 index 00000000000..766fe6d0924 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteExcludeHistoricalBalanceRocksDbTest.java @@ -0,0 +1,13 @@ +package org.tron.plugins.rocksdb; + +import java.io.IOException; +import org.junit.Test; +import org.tron.plugins.DbLiteTest; + +public class DbLiteExcludeHistoricalBalanceRocksDbTest extends DbLiteTest { + + @Test + public void testToolsWithExcludeHistoricalBalance() throws InterruptedException, IOException { + testTools("ROCKSDB", 1, true); + } +} From 85d44c2be4a74c391ff457e0e1f661d517b25289 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Thu, 7 May 2026 13:51:40 +0800 Subject: [PATCH 2/2] docs(toolkit): clarify lite snapshot trace exclusion --- plugins/README.md | 4 +-- .../java/common/org/tron/plugins/DbLite.java | 26 ++++++++++--------- .../java/org/tron/plugins/DbLiteTest.java | 6 +++-- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/plugins/README.md b/plugins/README.md index c405223cab3..f14e070c01a 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -75,7 +75,7 @@ DB lite provides lite database, parameters are compatible with previous `LiteFul - `-fn | --fn-data-path`: The database path to be split or merged. - `-ds | --dataset-path`: When operation is `split`,`dataset-path` is the path that store the `snapshot` or `history`, when operation is `split`, `dataset-path` is the `history` data path. -- `--exclude-historical-balance`: Only used with `operate=split -t snapshot`, default: false. When set to true, `balance-trace` and `account-trace` are excluded from the lite snapshot. The flag has functional impact only when the source full node ran with `historyBalanceLookup=true` (off by default; most operators are unaffected). **WARNING:** for nodes that had `historyBalanceLookup=true`, this loss is permanent — a lite node booted from such a snapshot cannot answer historical balance lookups (`getBlockBalance` / `getAccountBalance`), and running `merge` afterwards will NOT restore the feature. If you need historical balance lookup on the resulting lite node, do **not** enable this flag. `split -t history` and `merge` ignore this flag. +- `--exclude-historical-balance`: Only used with `operate=split -t snapshot`, default: false. When set to true, `balance-trace` and `account-trace` are excluded from the lite snapshot. The flag has functional impact only when the source full node ran with `historyBalanceLookup=true` (off by default; most operators are unaffected). **WARNING:** for nodes that had `historyBalanceLookup=true`, this loss is permanent — a lite node booted from such a snapshot cannot safely serve historical balance lookups (`getBlockBalance` may fail, and `getAccountBalance` may return `balance=0` when `account-trace` data is missing), and running `merge` afterwards will NOT restore the feature. If you need historical balance lookup on the resulting lite node, do **not** enable this flag. `split -t history` and `merge` ignore this flag. - `-h | --help`: Provide the help info. ### Examples: @@ -87,7 +87,7 @@ DB lite provides lite database, parameters are compatible with previous `LiteFul #split and get a snapshot dataset java -jar Toolkit.jar db lite -o split -t snapshot --fn-data-path output-directory/database --dataset-path /tmp #split and get a snapshot dataset without balance-trace / account-trace (smaller snapshot; - #historical balance lookup will be permanently unavailable on the resulting lite node) + #historical balance lookup cannot be safely served on the resulting lite node) java -jar Toolkit.jar db lite -o split -t snapshot --fn-data-path output-directory/database --dataset-path /tmp --exclude-historical-balance #split and get a history dataset java -jar Toolkit.jar db lite -o split -t history --fn-data-path output-directory/database --dataset-path /tmp diff --git a/plugins/src/main/java/common/org/tron/plugins/DbLite.java b/plugins/src/main/java/common/org/tron/plugins/DbLite.java index 2804cdc2800..1a7e4e270f7 100644 --- a/plugins/src/main/java/common/org/tron/plugins/DbLite.java +++ b/plugins/src/main/java/common/org/tron/plugins/DbLite.java @@ -120,11 +120,12 @@ enum Type { snapshot, history } + "This flag only has a functional impact when the source full node ran with " + "`historyBalanceLookup=true` (off by default; most operators are unaffected). " + "WARNING: when historyBalanceLookup was enabled, this loss is permanent: a lite " - + "node booted from such a snapshot cannot answer historical balance lookups " - + "(getBlockBalance / getAccountBalance), and running merge afterwards will NOT " - + "restore the feature. If you need to keep historyBalanceLookup working on the " - + "resulting lite node, do NOT enable this flag. `split -t history` and `merge` " - + "ignore this flag.", + + "node booted from such a snapshot cannot safely serve historical balance lookups " + + "(getBlockBalance may fail, and getAccountBalance may return balance=0 when " + + "account-trace data is missing). Running merge afterwards will NOT restore the " + + "feature. If you need to keep historyBalanceLookup working on the resulting " + + "lite node, do NOT enable this flag. `split -t history` and `merge` ignore " + + "this flag.", order = 5) private boolean excludeHistoricalBalance; @@ -143,8 +144,8 @@ public Integer call() { try { switch (this.operate) { case split: - warnIfExcludingHistoricalBalance(); if (Type.snapshot == this.type) { + warnIfExcludingHistoricalBalance(); generateSnapshot(fnDataPath, datasetPath); } else if (Type.history == type) { generateHistory(fnDataPath, datasetPath); @@ -307,12 +308,14 @@ private void warnIfExcludingHistoricalBalance() { + "will be excluded from the lite snapshot. This only matters when the source full " + "node ran with historyBalanceLookup=true (off by default; most operators are " + "unaffected). When that switch was enabled, this loss is permanent: lite nodes " - + "booted from this snapshot cannot answer historical balance lookups " - + "(getBlockBalance / getAccountBalance), and running merge afterwards will NOT " - + "restore the feature. If you need to keep historyBalanceLookup working on the " - + "resulting lite node, do NOT use this flag."; + + "booted from this snapshot cannot safely serve historical balance lookups " + + "(getBlockBalance may fail, and getAccountBalance may return balance=0 when " + + "account-trace data is missing). Running merge afterwards will NOT restore the " + + "feature. If you need to keep historyBalanceLookup working on the resulting " + + "lite node, do NOT use this flag."; logger.warn(msg); - spec.commandLine().getErr().println(msg); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .errorText(msg)); } private List getSnapshotDbs(String sourceDir) { @@ -786,4 +789,3 @@ public long getSnapshotMaxNum() { } - diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java index 7702cb046cc..f7cb7b7f74f 100644 --- a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java @@ -71,7 +71,7 @@ public void shutdown() throws InterruptedException { context.close(); } - public void init(String dbType) throws IOException { + public void init(String dbType, boolean historyBalanceLookup) throws IOException { dbPath = folder.newFolder().toString(); Args.setParam(new String[] { "-d", dbPath, "-w", "--p2p-disable", "true", "--storage-db-engine", dbType}, @@ -80,6 +80,7 @@ public void init(String dbType) throws IOException { Args.getInstance().setAllowAccountStateRoot(1); Args.getInstance().setRpcPort(PublicMethod.chooseRandomPort()); Args.getInstance().setRpcEnable(true); + Args.getInstance().setHistoryBalanceLookup(historyBalanceLookup); databaseDir = Args.getInstance().getStorage().getDbDirectory(); // init dbBackupConfig to avoid NPE Args.getInstance().dbBackupConfig = DbBackupConfig.getInstance(); @@ -99,7 +100,8 @@ public void testTools(String dbType, int checkpointVersion, boolean excludeHisto throws InterruptedException, IOException { logger.info("dbType {}, checkpointVersion {}, excludeHistoricalBalance {}", dbType, checkpointVersion, excludeHistoricalBalance); - init(dbType); + boolean historyBalanceLookup = excludeHistoricalBalance; + init(dbType, historyBalanceLookup); final String[] argsForSnapshot = excludeHistoricalBalance ? new String[] {"-o", "split", "-t", "snapshot", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path",