From f8ff2c02753358584ff12a57fbb9b26258e3e9e6 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 12 Feb 2026 19:00:32 +0800 Subject: [PATCH 1/5] fix(ci): speed up snapshot downloading on CI --- .github/workflows/forest.yml | 1 + docs/docs/users/reference/env_variables.md | 1 + scripts/tests/calibnet_kademlia_check.sh | 1 + scripts/tests/calibnet_no_discovery_check.sh | 1 + scripts/tests/calibnet_stateless_mode_check.sh | 1 + scripts/tests/harness.sh | 8 ++++++++ src/daemon/mod.rs | 15 +++++++++++++-- 7 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/forest.yml b/.github/workflows/forest.yml index 820a111e9630..37c539841dcb 100644 --- a/.github/workflows/forest.yml +++ b/.github/workflows/forest.yml @@ -36,6 +36,7 @@ env: CXX: sccache clang++ FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters SHELL_IMAGE: busybox + FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH: /var/tmp/forest_snapshot_calibnet_latest.forest.car.zst jobs: build-macos: name: Build MacOS diff --git a/docs/docs/users/reference/env_variables.md b/docs/docs/users/reference/env_variables.md index cf28d14de39c..3c399c85c3f5 100644 --- a/docs/docs/users/reference/env_variables.md +++ b/docs/docs/users/reference/env_variables.md @@ -56,6 +56,7 @@ process. | `FOREST_ETH_BLOCK_CACHE_SIZE` | positive integer | 500 | 1 | The size of Eth block cache | | `FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK` | 1 or true | false | 1 | Whether or not to backfill full tipsets from the p2p network | | `FOREST_STRICT_JSON` | 1 or true | false | 1 | Enable strict JSON validation to detect duplicate keys in RPC requests | +| `FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH` | URL or file path | empty | `/var/tmp/forest_snapshot_calibnet.forest.car.zst` | Override snapshot path for `--auto-download-snapshot` | ### `FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT` diff --git a/scripts/tests/calibnet_kademlia_check.sh b/scripts/tests/calibnet_kademlia_check.sh index db916312d6c1..59629301a88e 100755 --- a/scripts/tests/calibnet_kademlia_check.sh +++ b/scripts/tests/calibnet_kademlia_check.sh @@ -24,6 +24,7 @@ cat <<- EOF > $CONFIG_PATH kademlia = true EOF +handle_auto_download_snapshot_env $FOREST_PATH --chain calibnet --encrypt-keystore false --auto-download-snapshot --config "$CONFIG_PATH" --save-token ./admin_token --rpc-address 127.0.0.1:12345 --metrics-address 127.0.0.1:6117 --healthcheck-address 127.0.0.1:2347 & FOREST_NODE_PID=$! # Verify that more peers are connected via kademlia diff --git a/scripts/tests/calibnet_no_discovery_check.sh b/scripts/tests/calibnet_no_discovery_check.sh index 8d4c14f13ffb..9b5ff73ddc83 100755 --- a/scripts/tests/calibnet_no_discovery_check.sh +++ b/scripts/tests/calibnet_no_discovery_check.sh @@ -11,6 +11,7 @@ function shutdown { trap shutdown EXIT +handle_auto_download_snapshot_env $FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --exit-after-init $FOREST_PATH --chain calibnet --encrypt-keystore false --mdns false --kademlia false --auto-download-snapshot --log-dir "$LOG_DIRECTORY" & FOREST_NODE_PID=$! diff --git a/scripts/tests/calibnet_stateless_mode_check.sh b/scripts/tests/calibnet_stateless_mode_check.sh index 0d4b602e89f1..f4fa5d44e735 100755 --- a/scripts/tests/calibnet_stateless_mode_check.sh +++ b/scripts/tests/calibnet_stateless_mode_check.sh @@ -24,6 +24,7 @@ cat <<- EOF > $CONFIG_PATH kademlia = false EOF +handle_auto_download_snapshot_env # Disable discovery to not connect to more nodes $FOREST_PATH --chain calibnet --encrypt-keystore false --auto-download-snapshot --config "$CONFIG_PATH" --rpc false --metrics-address 127.0.0.1:6117 --healthcheck-address 127.0.0.1:2347 & FOREST_NODE_PID=$! diff --git a/scripts/tests/harness.sh b/scripts/tests/harness.sh index f949b7700ab6..6e7d00f531d2 100644 --- a/scripts/tests/harness.sh +++ b/scripts/tests/harness.sh @@ -16,6 +16,13 @@ LOG_DIRECTORY=$TMP_DIR/logs export TMP_DIR export LOG_DIRECTORY +function handle_auto_download_snapshot_env { + if [ -v FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH ]; then + echo "Downloading calibnet snapshot to ${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}" + aria2c -x5 -c https://forest-archive.chainsafe.dev/latest/calibnet/ -o "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}" + fi +} + function forest_import_non_calibnet_snapshot { echo "Importing a non calibnet snapshot" $FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --import-snapshot ./test-snapshots/chain4.car @@ -23,6 +30,7 @@ function forest_import_non_calibnet_snapshot { function forest_download_and_import_snapshot { echo "Downloading and importing snapshot" + handle_auto_download_snapshot_env $FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --height=-200 --auto-download-snapshot } diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index c753a239aed9..a836def9f13c 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -709,8 +709,19 @@ async fn maybe_set_snapshot_path( (false, _, _) => {} // noop - don't need a snapshot (true, true, _) => {} // noop - we need a snapshot, and we have one (true, false, true) => { - let url = crate::cli_shared::snapshot::stable_url(vendor, chain)?; - config.client.snapshot_path = Some(url.to_string().into()); + const AUTO_SNAPSHOT_PATH_ENV_KEY: &str = "FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH"; + match std::env::var(AUTO_SNAPSHOT_PATH_ENV_KEY) { + Ok(path) if !path.is_empty() => { + tracing::info!( + "importing snapshot from {path} set by `{AUTO_SNAPSHOT_PATH_ENV_KEY}`" + ); + config.client.snapshot_path = Some(path.into()); + } + _ => { + let url = crate::cli_shared::snapshot::stable_url(vendor, chain)?; + config.client.snapshot_path = Some(url.to_string().into()); + } + } } (true, false, false) => { // we need a snapshot, don't have one, and don't have permission to download one, so ask the user From 4227a6cef78a7d6921c76c3f656535d904e56457 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 12 Feb 2026 19:51:13 +0800 Subject: [PATCH 2/5] fix --- scripts/tests/harness.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/harness.sh b/scripts/tests/harness.sh index 6e7d00f531d2..fdad5ac5af01 100644 --- a/scripts/tests/harness.sh +++ b/scripts/tests/harness.sh @@ -19,7 +19,7 @@ export LOG_DIRECTORY function handle_auto_download_snapshot_env { if [ -v FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH ]; then echo "Downloading calibnet snapshot to ${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}" - aria2c -x5 -c https://forest-archive.chainsafe.dev/latest/calibnet/ -o "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}" + aria2c -x5 -c https://forest-archive.chainsafe.dev/latest/calibnet/ -d $(dirname "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}") -o $(basename "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}") fi } From d2acfb805d0d4a71f38ba271b2cb558a8561cfff Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 12 Feb 2026 20:29:22 +0800 Subject: [PATCH 3/5] Update scripts/tests/harness.sh Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- scripts/tests/harness.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/tests/harness.sh b/scripts/tests/harness.sh index fdad5ac5af01..333aba69d613 100644 --- a/scripts/tests/harness.sh +++ b/scripts/tests/harness.sh @@ -17,11 +17,15 @@ export TMP_DIR export LOG_DIRECTORY function handle_auto_download_snapshot_env { - if [ -v FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH ]; then + if [[ -n "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH:-}" ]]; then echo "Downloading calibnet snapshot to ${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}" - aria2c -x5 -c https://forest-archive.chainsafe.dev/latest/calibnet/ -d $(dirname "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}") -o $(basename "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}") + mkdir -p "$(dirname "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}")" + aria2c -x5 -c https://forest-archive.chainsafe.dev/latest/calibnet/ \ + -d "$(dirname "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}")" \ + -o "$(basename "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}")" fi } +} function forest_import_non_calibnet_snapshot { echo "Importing a non calibnet snapshot" From 3765c1b756879fb7aec205e4af3fd3e67f031df5 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 12 Feb 2026 20:34:10 +0800 Subject: [PATCH 4/5] fix --- scripts/tests/harness.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/tests/harness.sh b/scripts/tests/harness.sh index 333aba69d613..07704e23dbd1 100644 --- a/scripts/tests/harness.sh +++ b/scripts/tests/harness.sh @@ -25,7 +25,6 @@ function handle_auto_download_snapshot_env { -o "$(basename "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}")" fi } -} function forest_import_non_calibnet_snapshot { echo "Importing a non calibnet snapshot" From c137c550194693a02d99d8921843dd578bb1961d Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 13 Feb 2026 16:35:50 +0800 Subject: [PATCH 5/5] comment --- scripts/tests/harness.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/tests/harness.sh b/scripts/tests/harness.sh index 07704e23dbd1..1a1eb5489f9a 100644 --- a/scripts/tests/harness.sh +++ b/scripts/tests/harness.sh @@ -16,6 +16,9 @@ LOG_DIRECTORY=$TMP_DIR/logs export TMP_DIR export LOG_DIRECTORY +# Use `aria2` to fetch the latest calibnet snapshot +# when `FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH` is set +# This optimization is used on CI to speed up snapshot downloading function handle_auto_download_snapshot_env { if [[ -n "${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH:-}" ]]; then echo "Downloading calibnet snapshot to ${FOREST_AUTO_DOWNLOAD_SNAPSHOT_PATH}"