fix(test): speed up test-release and codecov on CI

.github/workflows/coverage.yml

@@ -28,20 +28,32 @@ env:
   CI: 1
   CARGO_INCREMENTAL: 0
   CACHE_TIMEOUT_MINUTES: 5
+  AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+  AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
   RUSTC_WRAPPER: sccache
   CC: sccache clang
   CXX: sccache clang++
   # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times
   RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld"
+  FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT: 1
+  FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters
   RUST_LOG: error
 
 jobs:
   codecov:
     name: Coverage
     if: github.event.pull_request.draft == false && github.actor != 'dependabot[bot]'
-    runs-on: buildjet-4vcpu-ubuntu-2204
+    runs-on: ubuntu-24.04-arm
     timeout-minutes: 45
     steps:
+      - name: Configure SCCache variables
+        run: |
+          # External PRs do not have access to 'vars' or 'secrets'.
+          if [[ "${{secrets.AWS_ACCESS_KEY_ID}}" != "" ]]; then
+            echo "SCCACHE_ENDPOINT=${{ vars.SCCACHE_ENDPOINT}}" >> $GITHUB_ENV
+            echo "SCCACHE_BUCKET=${{ vars.SCCACHE_BUCKET}}" >> $GITHUB_ENV
+            echo "SCCACHE_REGION=${{ vars.SCCACHE_REGION}}" >> $GITHUB_ENV
+          fi
       - uses: actions/checkout@v6
       - name: Setup sccache
         uses: mozilla-actions/sccache-action@v0.0.9
@@ -52,6 +64,10 @@ jobs:
           go-version-file: "go.work"
       - uses: taiki-e/install-action@cargo-llvm-cov
       - uses: taiki-e/install-action@nextest
+      - name: Fetch proof params and RPC test snapshots
+        run: |
+          cargo run --bin forest-dev --no-default-features --profile quick -- fetch-rpc-tests
+          ls -ahl $FIL_PROOFS_PARAMETER_CACHE
       - name: Generate code coverage
         run: make codecov
       # Save lcov.info as an artifact for debugging purposes

.github/workflows/unit-tests.yml

@@ -28,9 +28,12 @@ env:
   CI: 1
   CARGO_INCREMENTAL: 0
   CACHE_TIMEOUT_MINUTES: 5
+  AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+  AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
   RUSTC_WRAPPER: "sccache"
   CC: "sccache clang"
   CXX: "sccache clang++"
+  FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters
   RUST_LOG: error
 
 jobs:
@@ -40,6 +43,14 @@ jobs:
     # This is done to limit the runner cost.
     if: github.event.pull_request.draft == false
     steps:
+      - name: Configure SCCache variables
+        run: |
+          # External PRs do not have access to 'vars' or 'secrets'.
+          if [[ "${{secrets.AWS_ACCESS_KEY_ID}}" != "" ]]; then
+            echo "SCCACHE_ENDPOINT=${{ vars.SCCACHE_ENDPOINT}}" >> $GITHUB_ENV
+            echo "SCCACHE_BUCKET=${{ vars.SCCACHE_BUCKET}}" >> $GITHUB_ENV
+            echo "SCCACHE_REGION=${{ vars.SCCACHE_REGION}}" >> $GITHUB_ENV
+          fi
       # find the nearest S3 space for storing cache files
       - name: Show IP
         run: curl ifconfig.me
@@ -62,12 +73,17 @@ jobs:
           go-version-file: "go.work"
       - name: install nextest
         uses: taiki-e/install-action@nextest
+      - name: Fetch proof params and RPC test snapshots
+        run: |
+          cargo run --bin forest-dev --no-default-features --profile quick -- fetch-rpc-tests
+          ls -ahl $FIL_PROOFS_PARAMETER_CACHE
       - run: |
           make test-release-docs
           make test-release
         env:
           # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times
           RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld"
+          FOREST_TEST_SKIP_PROOF_PARAM_CHECK: 1
       - id: get-cache-hash
         run: |
           ls -lhR ~/.cache/forest/test/rpc-snapshots/rpc_test/*

Makefile

@@ -115,7 +115,7 @@ test-release-docs:
 	cargo test --profile quick --doc --features doctest-private
 
 codecov:
-	cargo llvm-cov --workspace --codecov --output-path lcov.info
+	cargo llvm-cov -p forest-filecoin --codecov --output-path lcov.info
 
 # Checks if all headers are present and adds if not
 license:

src/bin/forest-dev.rs

@@ -0,0 +1,7 @@
+// Copyright 2019-2025 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+#[tokio::main(flavor = "multi_thread")]
+async fn main() -> anyhow::Result<()> {
+    forest::forest_dev_main(std::env::args_os()).await
+}

src/dev/main.rs

@@ -0,0 +1,18 @@
+// Copyright 2019-2025 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+use super::subcommands::Cli;
+use crate::cli_shared::logger::setup_minimal_logger;
+use clap::Parser as _;
+use std::ffi::OsString;
+
+pub async fn main<ArgT>(args: impl IntoIterator<Item = ArgT>) -> anyhow::Result<()>
+where
+    ArgT: Into<OsString> + Clone,
+{
+    // Capture Cli inputs
+    let Cli { cmd } = Cli::parse_from(args);
+    setup_minimal_logger();
+    let client = crate::rpc::Client::default_or_from_env(None)?;
+    cmd.run(client).await
+}

src/dev/mod.rs

@@ -0,0 +1,5 @@
+// Copyright 2019-2025 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+pub mod main;
+pub mod subcommands;

src/dev/subcommands/mod.rs

@@ -0,0 +1,87 @@
+// Copyright 2019-2025 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+use crate::cli_shared::cli::HELP_MESSAGE;
+use crate::rpc::Client;
+use crate::utils::net::{DownloadFileOption, download_file_with_cache};
+use crate::utils::proofs_api::ensure_proof_params_downloaded;
+use crate::utils::version::FOREST_VERSION_STRING;
+use anyhow::Context as _;
+use clap::Parser;
+use directories::ProjectDirs;
+use std::borrow::Cow;
+use std::path::PathBuf;
+use std::time::Duration;
+use tokio::task::JoinSet;
+use url::Url;
+
+/// Command-line options for the `forest-dev` binary
+#[derive(Parser)]
+#[command(name = env!("CARGO_PKG_NAME"), bin_name = "forest-dev", author = env!("CARGO_PKG_AUTHORS"), version = FOREST_VERSION_STRING.as_str(), about = env!("CARGO_PKG_DESCRIPTION")
+)]
+#[command(help_template(HELP_MESSAGE))]
+pub struct Cli {
+    #[command(subcommand)]
+    pub cmd: Subcommand,
+}
+
+/// forest-dev sub-commands
+#[derive(clap::Subcommand)]
+pub enum Subcommand {
+    /// Fetch RPC test snapshots to the local cache
+    FetchRpcTests,
+}
+
+impl Subcommand {
+    pub async fn run(self, _client: Client) -> anyhow::Result<()> {
+        match self {
+            Self::FetchRpcTests => fetch_rpc_tests().await,
+        }
+    }
+}
+
+async fn fetch_rpc_tests() -> anyhow::Result<()> {
+    crate::utils::proofs_api::maybe_set_proofs_parameter_cache_dir_env(
+        &crate::Config::default().client.data_dir,
+    );
+    ensure_proof_params_downloaded().await?;
+    let tests = include_str!("../../tool/subcommands/api_cmd/test_snapshots.txt")
+        .lines()
+        .map(|i| {
+            // Remove comment
+            i.split("#").next().unwrap().trim().to_string()
+        })
+        .filter(|l| !l.is_empty() && !l.starts_with('#'));
+    let mut joinset = JoinSet::new();
+    for test in tests {
+        joinset.spawn(fetch_rpc_test_snapshot(test.into()));
+    }
+    for result in joinset.join_all().await {
+        if let Err(e) = result {
+            tracing::warn!("{e}");
+        }
+    }
+    Ok(())
+}
+
+pub async fn fetch_rpc_test_snapshot<'a>(name: Cow<'a, str>) -> anyhow::Result<PathBuf> {
+    let url: Url =
+        format!("https://forest-snapshots.fra1.cdn.digitaloceanspaces.com/rpc_test/{name}")
+            .parse()
+            .with_context(|| format!("Failed to parse URL for test: {name}"))?;
+    let project_dir =
+        ProjectDirs::from("com", "ChainSafe", "Forest").context("failed to get project dir")?;
+    let cache_dir = project_dir.cache_dir().join("test").join("rpc-snapshots");
+    let path = crate::utils::retry(
+        crate::utils::RetryArgs {
+            timeout: Some(Duration::from_secs(30)),
+            max_retries: Some(5),
+            delay: Some(Duration::from_secs(1)),
+        },
+        || download_file_with_cache(&url, &cache_dir, DownloadFileOption::NonResumable),
+    )
+    .await
+    .map_err(|e| anyhow::anyhow!("failed to fetch rpc test snapshot {name} :{e}"))?
+    .path;
+    Ok(path)
+}

src/lib.rs

@@ -47,6 +47,7 @@ mod cli;
 mod cli_shared;
 mod daemon;
 mod db;
+mod dev;
 mod documentation;
 mod eth;
 mod f3;
@@ -124,6 +125,7 @@ pub use auth::{JWT_IDENTIFIER, verify_token};
 pub use cli::main::main as forest_main;
 pub use cli_shared::cli::{Client, Config};
 pub use daemon::main::main as forestd_main;
+pub use dev::main::main as forest_dev_main;
 pub use key_management::{
     ENCRYPTED_KEYSTORE_NAME, FOREST_KEYSTORE_PHRASE_ENV, KEYSTORE_NAME, KeyStore, KeyStoreConfig,
 };

src/tool/subcommands/api_cmd/test_snapshot.rs

@@ -185,47 +185,24 @@ async fn ctx(
 mod tests {
     use super::*;
     use crate::Config;
-    use crate::utils::net::{DownloadFileOption, download_file_with_cache};
     use crate::utils::proofs_api::ensure_proof_params_downloaded;
     use ahash::HashSet;
-    use anyhow::Context as _;
-    use directories::ProjectDirs;
-    use std::sync::LazyLock;
-    use std::time::{Duration, Instant};
-    use tokio::sync::Mutex;
-    use url::Url;
+    use std::time::Instant;
 
     // To run a single test: cargo test --lib filecoin_multisig_statedecodeparams_1754230255631789 -- --nocapture
     include!(concat!(env!("OUT_DIR"), "/__rpc_regression_tests_gen.rs"));
 
     async fn rpc_regression_test_run(name: &str) {
         // Set proof parameter data dir and make sure the proofs are available
         {
-            static PROOF_PARAMS_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
-            let _guard = PROOF_PARAMS_LOCK.lock().await;
             crate::utils::proofs_api::maybe_set_proofs_parameter_cache_dir_env(
                 &Config::default().client.data_dir,
             );
             ensure_proof_params_downloaded().await.unwrap();
         }
-        let url: Url =
-            format!("https://forest-snapshots.fra1.cdn.digitaloceanspaces.com/rpc_test/{name}")
-                .parse()
-                .with_context(|| format!("Failed to parse URL for test: {name}"))
-                .unwrap();
-        let project_dir = ProjectDirs::from("com", "ChainSafe", "Forest").unwrap();
-        let cache_dir = project_dir.cache_dir().join("test").join("rpc-snapshots");
-        let path = crate::utils::retry(
-            crate::utils::RetryArgs {
-                timeout: Some(Duration::from_secs(30)),
-                max_retries: Some(5),
-                delay: Some(Duration::from_secs(1)),
-            },
-            || download_file_with_cache(&url, &cache_dir, DownloadFileOption::NonResumable),
-        )
-        .await
-        .unwrap()
-        .path;
+        let path = crate::dev::subcommands::fetch_rpc_test_snapshot(name.into())
+            .await
+            .unwrap();
 
         // We need to set RNG seed so that tests are run with deterministic
         // output. The snapshots should be generated with a node running with the same seed, if

src/utils/proofs_api/parameters.rs

@@ -89,9 +89,16 @@ pub(super) async fn check_parameter_file(path: &Path, info: &ParameterData) -> a
 // %DATA_DIR/filecoin-proof-parameters unless the FIL_PROOFS_PARAMETER_CACHE
 // environment variable is set.
 pub(super) fn param_dir(data_dir: &Path) -> PathBuf {
-    std::env::var(PathBuf::from(PROOFS_PARAMETER_CACHE_ENV))
-        .map(PathBuf::from)
-        .unwrap_or_else(|_| data_dir.join(PARAM_DIR))
+    std::env::var(PROOFS_PARAMETER_CACHE_ENV)
+        .ok()
+        .and_then(|v| {
+            if v.is_empty() {
+                None
+            } else {
+                Some(PathBuf::from(v))
+            }
+        })
+        .unwrap_or_else(|| data_dir.join(PARAM_DIR))
 }
 
 /// Forest uses a set of external crates for verifying the proofs generated by

src/utils/proofs_api/paramfetch.rs

@@ -10,7 +10,7 @@
 use std::{
     io::{self, ErrorKind},
     path::{Path, PathBuf},
-    sync::Arc,
+    sync::{Arc, LazyLock},
 };
 
 use crate::{
@@ -23,7 +23,10 @@ use crate::{
 use anyhow::{Context, bail};
 use backon::{ExponentialBuilder, Retryable};
 use futures::{AsyncWriteExt, TryStreamExt, stream::FuturesUnordered};
-use tokio::fs::{self};
+use tokio::{
+    fs::{self},
+    sync::Mutex,
+};
 use tracing::{debug, info, warn};
 
 use super::parameters::{
@@ -57,12 +60,24 @@ pub enum SectorSizeOpt {
 
 /// Ensures the parameter files are downloaded to cache dir
 pub async fn ensure_proof_params_downloaded() -> anyhow::Result<()> {
+    #[cfg(test)]
+    if is_env_truthy("FOREST_TEST_SKIP_PROOF_PARAM_CHECK") {
+        return Ok(());
+    }
+
     let data_dir = std::env::var(PROOFS_PARAMETER_CACHE_ENV).unwrap_or_default();
     if data_dir.is_empty() {
         anyhow::bail!("Proof parameter data dir is not set");
     }
-    get_params_default(Path::new(&data_dir), SectorSizeOpt::Keys, false).await?;
-    Ok(())
+    static RUN_ONCE: LazyLock<Mutex<bool>> = LazyLock::new(|| Mutex::new(false));
+    let mut run_once = RUN_ONCE.lock().await;
+    if *run_once {
+        Ok(())
+    } else {
+        get_params_default(Path::new(&data_dir), SectorSizeOpt::Keys, false).await?;
+        *run_once = true;
+        Ok(())
+    }
 }
 
 /// Get proofs parameters and all verification keys for a given sector size

src/utils/rand/mod.rs

@@ -35,6 +35,7 @@ fn forest_rng_internal(mode: ForestRngMode) -> impl Rng + CryptoRng {
     const ENV: &str = FIXED_RNG_SEED_ENV;
     if let Ok(v) = std::env::var(ENV) {
         if let Ok(seed) = v.parse() {
+            #[cfg(not(test))]
             tracing::warn!("[security] using test RNG with fixed seed {seed} set by {ENV}");
             return Either::Left(rand_chacha::ChaChaRng::seed_from_u64(seed));
         } else {

tests/common/mod.rs

@@ -41,7 +41,10 @@ impl CommonEnv for Command {
     // Always downloads proofs to same location to lower the overall test time
     // (by reducing multiple "fetching param file" steps).
     fn common_env(&mut self) -> &mut Self {
-        self.env("FIL_PROOFS_PARAMETER_CACHE", "/tmp/forest-test-fil-proofs")
+        match std::env::var("FIL_PROOFS_PARAMETER_CACHE").ok() {
+            Some(v) if !v.is_empty() => self,
+            _ => self.env("FIL_PROOFS_PARAMETER_CACHE", "/tmp/forest-test-fil-proofs"),
+        }
     }
 }
 

tests/config.rs

@@ -55,6 +55,7 @@ fn test_download_location_of_proof_parameter_files_default() {
 
     tool()
         .env("FOREST_CONFIG_PATH", config_file.path())
+        .env("FIL_PROOFS_PARAMETER_CACHE", "")
         .arg("fetch-params")
         .arg("--keys")
         .arg("--dry-run")