From ddb8aa85164d7fddc3021da7cee5d807c4804aa6 Mon Sep 17 00:00:00 2001 From: m0ar Date: Fri, 21 Mar 2025 09:40:11 +0100 Subject: [PATCH 1/4] docs: clarify only the first valid ETHEREUM RPC URL passed in config will be used for validation --- one/src/daemon.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/one/src/daemon.rs b/one/src/daemon.rs index 4898613cc..6dcaf4a00 100644 --- a/one/src/daemon.rs +++ b/one/src/daemon.rs @@ -240,6 +240,8 @@ pub struct DaemonOpts { anchor_poll_retry_count: u64, /// Ethereum RPC URLs used for time events validation. Required when connecting to mainnet and uses fallback URLs if not specified for other networks. + /// + /// Note: only the first valid RPC URL will be used by the time event validator #[arg( long, use_value_delimiter = true, From 8a9b2ba8cbe7c8b841fabfcb3b939d6c08e85e74 Mon Sep 17 00:00:00 2001 From: m0ar Date: Fri, 21 Mar 2025 10:53:23 +0100 Subject: [PATCH 2/4] docs: warn when IPFS migration fails to find blocks in input list, improve help texts --- one/src/migrations.rs | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/one/src/migrations.rs b/one/src/migrations.rs index 3ea92c622..e5f224562 100644 --- a/one/src/migrations.rs +++ b/one/src/migrations.rs @@ -13,7 +13,7 @@ use clap::{Args, Subcommand}; use futures::{stream::BoxStream, StreamExt}; use multihash_codetable::{Code, Multihash, MultihashDigest}; use tokio::io::AsyncBufReadExt; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use crate::{default_directory, DBOpts, Info, LogOpts}; @@ -66,13 +66,14 @@ pub struct FromIpfsOpts { #[command(flatten)] log_opts: LogOpts, - /// Path of file containing list of newline-delimited file paths to migrate. + /// Path of file containing list of newline-delimited paths to blocks to migrate. The paths must either be absolute, + /// or relative to the current directory where the migration command is run. /// /// See below for example usage when running a migration for a live IPFS node. Multiple migration runs using lists /// of files that have changed between runs is useful for incremental migrations. This method can also be used for /// a final migration after shutting down the IPFS node so that all inflight blocks are migrated to the new C1 node. /// - /// # Get list of files in sorted order + /// # Get list of absolute file paths to blocks, in sorted order /// /// find ~/.ipfs/blocks -type f | sort > first_run_files.txt /// @@ -95,6 +96,8 @@ pub struct FromIpfsOpts { input_file_list_path: Option, /// Offset within the input files to start from + /// + /// Note: this is useful for resuming a migration from a previous run that was interrupted. #[clap( long, short = 's', @@ -109,6 +112,9 @@ pub struct FromIpfsOpts { /// Optional list of model stream ids. Only events from these models will be migrated. /// If the list is empty all events are migrated. + /// + /// Note: if filtering, you likely also want to include the metamodel stream id + /// (kh4q0ozorrgaq2mezktnrmdwleo1d), so the actual model streams are also included in the migration. #[clap(long, value_delimiter = ',', env = "CERAMIC_ONE_MODEL_FILTER")] model_filter: Vec, @@ -118,7 +124,10 @@ pub struct FromIpfsOpts { validate_signatures: bool, /// Whether to validate the chain of time events. - /// Events with invalid chains will be skipped and counted as errors. + /// + /// Events with proofs from invalid chains will be skipped and counted as errors. + /// For example if some blocks being imported were anchored against an inmemory + /// store or a different network than configured with the `--network` option. #[clap(long, env = "CERAMIC_ONE_VALIDATE_CHAINS")] validate_chain: bool, } @@ -322,17 +331,26 @@ impl FSBlockStore { async fn block_from_path(block_path: PathBuf) -> Result)>> { if !block_path.is_file() { + if block_path.is_absolute() { + warn!( + path = %block_path.display(), + "block file not found, skipping"); + } else { + warn!( + path = %block_path.display(), + "block file not found at relative path, skipping"); + } return Ok(None); } let Ok((_base, hash_bytes)) = multibase::decode("B".to_string() + block_path.file_stem().unwrap().to_str().unwrap()) else { - debug!(path = %block_path.display(), "block filename is not valid base32upper"); + warn!(path = %block_path.display(), "block filename is not valid base32upper, skipping"); return Ok(None); }; let Ok(hash) = Multihash::from_bytes(&hash_bytes) else { - debug!(path = %block_path.display(), "block filename is not a valid multihash"); + warn!(path = %block_path.display(), "block filename is not a valid multihash, skipping"); return Ok(None); }; let blob = tokio::fs::read(&block_path).await?; From c2f04cb43a8225fe1ce6e895c1226b297cdc370d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edvard=20H=C3=BCbinette?= Date: Mon, 26 May 2025 15:44:55 +0200 Subject: [PATCH 3/4] migrations: clarify warnings and CLI docs Co-authored-by: Mohsin Zaidi <2236875+smrz2001@users.noreply.github.com> --- one/src/daemon.rs | 2 +- one/src/migrations.rs | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/one/src/daemon.rs b/one/src/daemon.rs index 6dcaf4a00..bb87cfbd0 100644 --- a/one/src/daemon.rs +++ b/one/src/daemon.rs @@ -241,7 +241,7 @@ pub struct DaemonOpts { /// Ethereum RPC URLs used for time events validation. Required when connecting to mainnet and uses fallback URLs if not specified for other networks. /// - /// Note: only the first valid RPC URL will be used by the time event validator + /// Note: only the first valid RPC URL for a particular chain will be used by the time event validator #[arg( long, use_value_delimiter = true, diff --git a/one/src/migrations.rs b/one/src/migrations.rs index e5f224562..938efa060 100644 --- a/one/src/migrations.rs +++ b/one/src/migrations.rs @@ -331,15 +331,7 @@ impl FSBlockStore { async fn block_from_path(block_path: PathBuf) -> Result)>> { if !block_path.is_file() { - if block_path.is_absolute() { - warn!( - path = %block_path.display(), - "block file not found, skipping"); - } else { - warn!( - path = %block_path.display(), - "block file not found at relative path, skipping"); - } + warn!(path = %block_path.display(), relative_path = %!block_path.is_absolute(), "block file not found, skipping"); return Ok(None); } From ccbf8eb227b8404339b6d6fa5cad984dd39569bf Mon Sep 17 00:00:00 2001 From: Mohsin Zaidi Date: Mon, 26 May 2025 21:49:04 +0530 Subject: [PATCH 4/4] chore: lint --- one/src/daemon.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/one/src/daemon.rs b/one/src/daemon.rs index 71f3c43d1..efebe38b2 100644 --- a/one/src/daemon.rs +++ b/one/src/daemon.rs @@ -239,7 +239,6 @@ pub struct DaemonOpts { anchor_poll_retry_count: u64, /// Ethereum RPC URLs used for time events validation. Required when connecting to mainnet and uses fallback URLs if not specified for other networks. - /// /// Note: only the first valid RPC URL for a particular chain will be used by the time event validator #[arg( long,