feat(db,lite): add --exclude-historical-balance for lite snapshot

plugins/README.md

@@ -75,16 +75,20 @@ DB lite provides lite database, parameters are compatible with previous `LiteFul
 - `-fn | --fn-data-path`: The database path to be split or merged.
 - `-ds | --dataset-path`: When operation is `split`,`dataset-path` is the path that store the `snapshot` or `history`, when
   operation is `split`, `dataset-path` is the `history` data path.
+- `--exclude-historical-balance`: Only used with `operate=split -t snapshot`, default: false. When set to true, `balance-trace` and `account-trace` are excluded from the lite snapshot. The flag has functional impact only when the source full node ran with `historyBalanceLookup=true` (off by default; most operators are unaffected). **WARNING:** for nodes that had `historyBalanceLookup=true`, this loss is permanent — a lite node booted from such a snapshot cannot safely serve historical balance lookups (`getBlockBalance` may fail, and `getAccountBalance` may return `balance=0` when `account-trace` data is missing), and running `merge` afterwards will NOT restore the feature. If you need historical balance lookup on the resulting lite node, do **not** enable this flag. `split -t history` and `merge` ignore this flag.
 - `-h | --help`: Provide the help info.
 
 ### Examples:
 
 ```shell script
 # full command
-  java -jar Toolkit.jar db lite [-h] -ds=<datasetPath> -fn=<fnDataPath> [-o=<operate>] [-t=<type>]
+  java -jar Toolkit.jar db lite [-h] -ds=<datasetPath> -fn=<fnDataPath> [-o=<operate>] [-t=<type>] [--exclude-historical-balance]
 # examples
   #split and get a snapshot dataset
   java -jar Toolkit.jar db lite -o split -t snapshot --fn-data-path output-directory/database --dataset-path /tmp
+  #split and get a snapshot dataset without balance-trace / account-trace (smaller snapshot;
+  #historical balance lookup cannot be safely served on the resulting lite node)
+  java -jar Toolkit.jar db lite -o split -t snapshot --fn-data-path output-directory/database --dataset-path /tmp --exclude-historical-balance
   #split and get a history dataset
   java -jar Toolkit.jar db lite -o split -t history --fn-data-path output-directory/database --dataset-path /tmp
   #merge history dataset and snapshot dataset

plugins/src/main/java/common/org/tron/plugins/DbLite.java

@@ -20,6 +20,7 @@
 import java.util.concurrent.Callable;
 import java.util.stream.Collectors;
 import java.util.stream.LongStream;
+import java.util.stream.Stream;
 import lombok.extern.slf4j.Slf4j;
 import me.tongfei.progressbar.ProgressBar;
 import org.rocksdb.RocksDBException;
@@ -57,6 +58,8 @@ public class DbLite implements Callable<Integer> {
   private static final String TRANSACTION_HISTORY_DB_NAME = "transactionHistoryStore";
   private static final String PROPERTIES_DB_NAME = "properties";
   private static final String TRANS_CACHE_DB_NAME = "trans-cache";
+  private static final String BALANCE_TRACE_DB_NAME = "balance-trace";
+  private static final String ACCOUNT_TRACE_DB_NAME = "account-trace";
 
   private static final List<String> archiveDbs = Arrays.asList(
       BLOCK_DB_NAME,
@@ -65,6 +68,10 @@ public class DbLite implements Callable<Integer> {
       TRANSACTION_RET_DB_NAME,
       TRANSACTION_HISTORY_DB_NAME);
 
+  private static final List<String> traceDbs = Arrays.asList(
+      BALANCE_TRACE_DB_NAME,
+      ACCOUNT_TRACE_DB_NAME);
+
   enum Operate { split, merge }
 
   enum Type { snapshot, history }
@@ -105,8 +112,26 @@ enum Type { snapshot, history }
   private String datasetPath;
 
   @CommandLine.Option(
-      names = {"--help", "-h"},
+      names = {"--exclude-historical-balance"},
+      defaultValue = "false",
+      description = "only used with `operate=split -t snapshot`: when true, balance-trace "
+          + "and account-trace are excluded from the lite snapshot. "
+          + "Default: ${DEFAULT-VALUE} (legacy behavior; trace stores stay in the snapshot). "
+          + "This flag only has a functional impact when the source full node ran with "
+          + "`historyBalanceLookup=true` (off by default; most operators are unaffected). "
+          + "WARNING: when historyBalanceLookup was enabled, this loss is permanent: a lite "
+          + "node booted from such a snapshot cannot safely serve historical balance lookups "
+          + "(getBlockBalance may fail, and getAccountBalance may return balance=0 when "
+          + "account-trace data is missing). Running merge afterwards will NOT restore the "
+          + "feature. If you need to keep historyBalanceLookup working on the resulting "
+          + "lite node, do NOT enable this flag. `split -t history` and `merge` ignore "
+          + "this flag.",
       order = 5)
+  private boolean excludeHistoricalBalance;
+
+  @CommandLine.Option(
+      names = {"--help", "-h"},
+      order = 6)
   private boolean help;
 
 
@@ -120,6 +145,7 @@ public Integer call() {
       switch (this.operate) {
         case split:
           if (Type.snapshot == this.type) {
+            warnIfExcludingHistoricalBalance();
             generateSnapshot(fnDataPath, datasetPath);
           } else if (Type.history == type) {
             generateHistory(fnDataPath, datasetPath);
@@ -253,12 +279,52 @@ public void completeHistoryData(String historyDir, String liteDir) {
     spec.commandLine().getOut().format("Merge history finished, take %d s.", during).println();
   }
 
+  /**
+   * Compute the directories to exclude from the lite snapshot.
+   * <p>
+   * Default ({@code --exclude-historical-balance=false}): the legacy archive set
+   * (5 dbs); {@link #BALANCE_TRACE_DB_NAME} / {@link #ACCOUNT_TRACE_DB_NAME}
+   * stay with the snapshot as state-style stores.
+   * <p>
+   * Opt-in ({@code --exclude-historical-balance=true}): the trace stores are
+   * additionally excluded, producing a smaller lite snapshot at the cost of
+   * dropping historical balance lookup support on the resulting lite node.
+   * Only {@code split -t snapshot} consults this. {@code split -t history}
+   * and {@code merge} always use the legacy archive set.
+   */
+  private List<String> snapshotExclusion() {
+    if (!excludeHistoricalBalance) {
+      return archiveDbs;
+    }
+    return Stream.concat(archiveDbs.stream(), traceDbs.stream())
+        .collect(Collectors.toList());
+  }
+
+  private void warnIfExcludingHistoricalBalance() {
+    if (!excludeHistoricalBalance) {
+      return;
+    }
+    String msg = "WARNING: --exclude-historical-balance is enabled. balance-trace / account-trace "
+        + "will be excluded from the lite snapshot. This only matters when the source full "
+        + "node ran with historyBalanceLookup=true (off by default; most operators are "
+        + "unaffected). When that switch was enabled, this loss is permanent: lite nodes "
+        + "booted from this snapshot cannot safely serve historical balance lookups "
+        + "(getBlockBalance may fail, and getAccountBalance may return balance=0 when "
+        + "account-trace data is missing). Running merge afterwards will NOT restore the "
+        + "feature. If you need to keep historyBalanceLookup working on the resulting "
+        + "lite node, do NOT use this flag.";
+    logger.warn(msg);
+    spec.commandLine().getErr().println(spec.commandLine().getColorScheme()
+        .errorText(msg));
+  }
+
   private List<String> getSnapshotDbs(String sourceDir) {
     List<String> snapshotDbs = Lists.newArrayList();
     File basePath = new File(sourceDir);
+    List<String> excluded = snapshotExclusion();
     Arrays.stream(Objects.requireNonNull(basePath.listFiles()))
             .filter(File::isDirectory)
-            .filter(dir -> !archiveDbs.contains(dir.getName()))
+            .filter(dir -> !excluded.contains(dir.getName()))
             .forEach(dir -> snapshotDbs.add(dir.getName()));
     return snapshotDbs;
   }
@@ -723,4 +789,3 @@ public long getSnapshotMaxNum() {
 }
 
 
-

plugins/src/test/java/org/tron/plugins/DbLiteTest.java

@@ -1,11 +1,14 @@
 package org.tron.plugins;
 
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.tron.common.utils.PublicMethod.getRandomPrivateKey;
 
 import io.grpc.ManagedChannel;
 import io.grpc.ManagedChannelBuilder;
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.After;
@@ -68,7 +71,7 @@ public void shutdown() throws InterruptedException {
     context.close();
   }
 
-  public void init(String dbType) throws IOException {
+  public void init(String dbType, boolean historyBalanceLookup) throws IOException {
     dbPath = folder.newFolder().toString();
     Args.setParam(new String[] {
         "-d", dbPath, "-w", "--p2p-disable", "true", "--storage-db-engine", dbType},
@@ -77,6 +80,7 @@ public void init(String dbType) throws IOException {
     Args.getInstance().setAllowAccountStateRoot(1);
     Args.getInstance().setRpcPort(PublicMethod.chooseRandomPort());
     Args.getInstance().setRpcEnable(true);
+    Args.getInstance().setHistoryBalanceLookup(historyBalanceLookup);
     databaseDir = Args.getInstance().getStorage().getDbDirectory();
     // init dbBackupConfig to avoid NPE
     Args.getInstance().dbBackupConfig = DbBackupConfig.getInstance();
@@ -89,10 +93,20 @@ public void clear() {
 
   public void testTools(String dbType, int checkpointVersion)
       throws InterruptedException, IOException {
-    logger.info("dbType {}, checkpointVersion {}", dbType, checkpointVersion);
-    init(dbType);
-    final String[] argsForSnapshot =
-        new String[] {"-o", "split", "-t", "snapshot", "--fn-data-path",
+    testTools(dbType, checkpointVersion, false);
+  }
+
+  public void testTools(String dbType, int checkpointVersion, boolean excludeHistoricalBalance)
+      throws InterruptedException, IOException {
+    logger.info("dbType {}, checkpointVersion {}, excludeHistoricalBalance {}",
+        dbType, checkpointVersion, excludeHistoricalBalance);
+    boolean historyBalanceLookup = excludeHistoricalBalance;
+    init(dbType, historyBalanceLookup);
+    final String[] argsForSnapshot = excludeHistoricalBalance
+        ? new String[] {"-o", "split", "-t", "snapshot", "--fn-data-path",
+            dbPath + File.separator + databaseDir, "--dataset-path",
+            dbPath, "--exclude-historical-balance"}
+        : new String[] {"-o", "split", "-t", "snapshot", "--fn-data-path",
             dbPath + File.separator + databaseDir, "--dataset-path",
             dbPath};
     final String[] argsForHistory =
@@ -114,6 +128,16 @@ public void testTools(String dbType, int checkpointVersion)
     FileUtil.deleteDir(Paths.get(dbPath, databaseDir, "trans-cache").toFile());
     // generate snapshot
     cli.execute(argsForSnapshot);
+    Path snapshotDir = Paths.get(dbPath, "snapshot");
+    if (excludeHistoricalBalance) {
+      // when --exclude-historical-balance=true, the lite snapshot must not ship
+      // balance-trace / account-trace
+      assertFalse(snapshotDir.resolve("balance-trace").toFile().exists());
+      assertFalse(snapshotDir.resolve("account-trace").toFile().exists());
+    } else {
+      assertTrue(snapshotDir.resolve("balance-trace").toFile().exists());
+      assertTrue(snapshotDir.resolve("account-trace").toFile().exists());
+    }
     // start fullNode
     startApp();
     // produce transactions

plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteExcludeHistoricalBalanceRocksDbTest.java

@@ -0,0 +1,13 @@
+package org.tron.plugins.rocksdb;
+
+import java.io.IOException;
+import org.junit.Test;
+import org.tron.plugins.DbLiteTest;
+
+public class DbLiteExcludeHistoricalBalanceRocksDbTest extends DbLiteTest {
+
+  @Test
+  public void testToolsWithExcludeHistoricalBalance() throws InterruptedException, IOException {
+    testTools("ROCKSDB", 1, true);
+  }
+}