From 3ea270a02a6a0bb7c212ecbc10c345c02c7a408f Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 31 Mar 2026 19:55:11 +0800 Subject: [PATCH 1/3] tool: forest-dev export-tipset-lookup --- src/db/memory.rs | 13 +++ .../subcommands/export_tipset_lookup_cmd.rs | 106 ++++++++++++++++++ src/dev/subcommands/mod.rs | 3 + 3 files changed, 122 insertions(+) create mode 100644 src/dev/subcommands/export_tipset_lookup_cmd.rs diff --git a/src/db/memory.rs b/src/db/memory.rs index 4dc636a146c6..45715e90651c 100644 --- a/src/db/memory.rs +++ b/src/db/memory.rs @@ -13,6 +13,7 @@ use anyhow::Context as _; use cid::Cid; use fvm_ipld_blockstore::Blockstore; use itertools::Itertools; +use nunny::Vec as NonEmpty; use parking_lot::RwLock; #[derive(Debug, Default)] @@ -24,6 +25,10 @@ pub struct MemoryDB { } impl MemoryDB { + pub fn blockstore_len(&self) -> usize { + self.blockchain_db.read().len() + self.blockchain_persistent_db.read().len() + } + pub fn blockstore_size_bytes(&self) -> usize { self.blockchain_db .read() @@ -41,6 +46,14 @@ impl MemoryDB { SettingsStoreExt::read_obj::(self, crate::db::setting_keys::HEAD_KEY)? .context("chain head is not tracked and cannot be exported")? .into_cids(); + self.export_forest_car_with_roots(roots, writer).await + } + + pub async fn export_forest_car_with_roots( + &self, + roots: NonEmpty, + writer: &mut W, + ) -> anyhow::Result<()> { let blocks = { let blockchain_db = self.blockchain_db.read(); let blockchain_persistent_db = self.blockchain_persistent_db.read(); diff --git a/src/dev/subcommands/export_tipset_lookup_cmd.rs b/src/dev/subcommands/export_tipset_lookup_cmd.rs new file mode 100644 index 000000000000..4d7f317c6cd9 --- /dev/null +++ b/src/dev/subcommands/export_tipset_lookup_cmd.rs @@ -0,0 +1,106 @@ +// Copyright 2019-2026 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::{ + chain::ChainStore, + cli_shared::{chain_path, read_config}, + daemon::db_util::load_all_forest_cars, + db::{ + CAR_DB_DIR_NAME, MemoryDB, + car::ManyCar, + db_engine::{db_root, open_db}, + }, + genesis::read_genesis_header, + networks::{ChainConfig, NetworkChain}, + shim::clock::ChainEpoch, +}; +use clap::Args; +use fil_actors_shared::fvm_ipld_amt::Amt; +use human_repr::HumanCount; +use std::{path::PathBuf, sync::Arc}; + +/// Exports epoch to tipset key mapping AMT as a `ForestCAR` file for a given epoch range. +/// The exported AMT can be used to quickly look up the tipset key for a given epoch without traversing the chain, +/// which is useful for tools that need to access historical tipsets frequently. +#[derive(Debug, Args)] +pub struct ExportTipsetLookupCommand { + /// Filecoin network chain (e.g., calibnet, mainnet) + #[arg(long, required = true)] + chain: NetworkChain, + /// Optional path to the database folder + #[arg(long)] + db: Option, + /// Start epoch (inclusive). Defaults to the current chain head + #[arg(long)] + from: Option, + /// End epoch (inclusive). Defaults to 0 (genesis) + #[arg(long)] + to: Option, + /// The path to the output `ForestCAR` file + #[arg(short, long)] + output: PathBuf, +} + +impl ExportTipsetLookupCommand { + pub async fn run(self) -> anyhow::Result<()> { + let Self { + chain, + db, + from, + to, + output, + } = self; + let db_root_path = if let Some(db) = db { + db + } else { + let (_, config) = read_config(None, Some(chain.clone()))?; + db_root(&chain_path(&config))? + }; + let forest_car_db_dir = db_root_path.join(CAR_DB_DIR_NAME); + let db = Arc::new(ManyCar::new(open_db(db_root_path, &Default::default())?)); + load_all_forest_cars(&db, &forest_car_db_dir)?; + + let chain_config = Arc::new(ChainConfig::from_chain(&chain)); + let genesis_header = + read_genesis_header(None, chain_config.genesis_bytes(&db).await?.as_deref(), &db) + .await?; + let chain_store = Arc::new(ChainStore::new( + db.clone(), + db.clone(), + db.clone(), + chain_config, + genesis_header, + )?); + + let head = chain_store.heaviest_tipset(); + + let amt_db = Arc::new(MemoryDB::default()); + let mut amt = Amt::new(&amt_db); + for ts in head.chain(chain_store.blockstore()) { + if let Some(from) = from + && ts.epoch() > from + { + continue; + } + if let Some(to) = to + && ts.epoch() < to + { + break; + } + amt.set(ts.epoch() as u64, ts.key().clone())?; + } + let root = amt.flush()?; + println!( + "Exported tipset lookup AMT with root CID: {root}, len: {}, size: {}", + amt_db.blockstore_len(), + amt_db.blockstore_size_bytes().human_count_bytes() + ); + amt_db + .export_forest_car_with_roots( + nunny::vec![root], + &mut tokio::fs::File::create(output).await?, + ) + .await?; + Ok(()) + } +} diff --git a/src/dev/subcommands/mod.rs b/src/dev/subcommands/mod.rs index 9f7fbf42e858..c880f1c4445c 100644 --- a/src/dev/subcommands/mod.rs +++ b/src/dev/subcommands/mod.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0, MIT mod archive_missing_cmd; +mod export_tipset_lookup_cmd; mod state_cmd; mod update_checkpoints_cmd; @@ -49,6 +50,7 @@ pub enum Subcommand { UpdateCheckpoints(update_checkpoints_cmd::UpdateCheckpointsCommand), /// Find missing archival snapshots on the Forest Archive for a given epoch range ArchiveMissing(archive_missing_cmd::ArchiveMissingCommand), + ExportTipsetLookup(export_tipset_lookup_cmd::ExportTipsetLookupCommand), } impl Subcommand { @@ -58,6 +60,7 @@ impl Subcommand { Self::State(cmd) => cmd.run().await, Self::UpdateCheckpoints(cmd) => cmd.run().await, Self::ArchiveMissing(cmd) => cmd.run().await, + Self::ExportTipsetLookup(cmd) => cmd.run().await, } } } From a52825d5dbff8a8de76746103c5e425fe4b44768 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 31 Mar 2026 20:31:09 +0800 Subject: [PATCH 2/3] add --skip-length --- src/dev/subcommands/export_tipset_lookup_cmd.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/dev/subcommands/export_tipset_lookup_cmd.rs b/src/dev/subcommands/export_tipset_lookup_cmd.rs index 4d7f317c6cd9..ec002da94919 100644 --- a/src/dev/subcommands/export_tipset_lookup_cmd.rs +++ b/src/dev/subcommands/export_tipset_lookup_cmd.rs @@ -17,7 +17,7 @@ use crate::{ use clap::Args; use fil_actors_shared::fvm_ipld_amt::Amt; use human_repr::HumanCount; -use std::{path::PathBuf, sync::Arc}; +use std::{num::NonZeroUsize, path::PathBuf, sync::Arc, time::Instant}; /// Exports epoch to tipset key mapping AMT as a `ForestCAR` file for a given epoch range. /// The exported AMT can be used to quickly look up the tipset key for a given epoch without traversing the chain, @@ -36,6 +36,9 @@ pub struct ExportTipsetLookupCommand { /// End epoch (inclusive). Defaults to 0 (genesis) #[arg(long)] to: Option, + /// Every N epochs to skip when exporting the AMT. Defaults to 1 (export every epoch) + #[arg(long, default_value = "1")] + skip_length: NonZeroUsize, /// The path to the output `ForestCAR` file #[arg(short, long)] output: PathBuf, @@ -48,8 +51,10 @@ impl ExportTipsetLookupCommand { db, from, to, + skip_length, output, } = self; + let skip_length = skip_length.get() as i64; let db_root_path = if let Some(db) = db { db } else { @@ -76,6 +81,7 @@ impl ExportTipsetLookupCommand { let amt_db = Arc::new(MemoryDB::default()); let mut amt = Amt::new(&amt_db); + let start = Instant::now(); for ts in head.chain(chain_store.blockstore()) { if let Some(from) = from && ts.epoch() > from @@ -87,13 +93,17 @@ impl ExportTipsetLookupCommand { { break; } + if ts.epoch() % skip_length != 0 { + continue; + } amt.set(ts.epoch() as u64, ts.key().clone())?; } let root = amt.flush()?; println!( - "Exported tipset lookup AMT with root CID: {root}, len: {}, size: {}", + "Exported tipset lookup AMT with root CID: {root}, len: {}, size: {}, took {}", amt_db.blockstore_len(), - amt_db.blockstore_size_bytes().human_count_bytes() + amt_db.blockstore_size_bytes().human_count_bytes(), + humantime::format_duration(start.elapsed()) ); amt_db .export_forest_car_with_roots( From 93db2d575023e50a7db327701d5a37a5fcb00e2e Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 1 Apr 2026 15:17:11 +0800 Subject: [PATCH 3/3] make to non-optional --- src/dev/subcommands/export_tipset_lookup_cmd.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/dev/subcommands/export_tipset_lookup_cmd.rs b/src/dev/subcommands/export_tipset_lookup_cmd.rs index ec002da94919..6e1593c5969c 100644 --- a/src/dev/subcommands/export_tipset_lookup_cmd.rs +++ b/src/dev/subcommands/export_tipset_lookup_cmd.rs @@ -33,9 +33,9 @@ pub struct ExportTipsetLookupCommand { /// Start epoch (inclusive). Defaults to the current chain head #[arg(long)] from: Option, - /// End epoch (inclusive). Defaults to 0 (genesis) - #[arg(long)] - to: Option, + /// End epoch (inclusive). + #[arg(long, default_value = "0")] + to: ChainEpoch, /// Every N epochs to skip when exporting the AMT. Defaults to 1 (export every epoch) #[arg(long, default_value = "1")] skip_length: NonZeroUsize, @@ -88,9 +88,7 @@ impl ExportTipsetLookupCommand { { continue; } - if let Some(to) = to - && ts.epoch() < to - { + if ts.epoch() < to { break; } if ts.epoch() % skip_length != 0 {