feat: implement storage command functionalities and enhance CLI operations

- Added new commands for storage operations including `cp`, `ls`, `cat`, `mv`, and `stat`, allowing users to manage files and objects in storage.
- Introduced a `client` module for HTTP interactions with the storage gateway API, facilitating blob uploads and downloads.
- Enhanced `StorageConfig` to support dynamic gateway URL retrieval from various sources.
- Implemented path parsing for storage URIs, improving the handling of storage paths.
- Updated `Cargo.toml` to include new dependencies for enhanced functionality.
- Added comprehensive error handling and user feedback for storage operations.

Author: phutchins

Cargo.lock

@@ -88,8 +88,13 @@ fendermint_app_settings = { path = "../../fendermint/app/settings" }
 fendermint_vm_actor_interface = { path = "../../fendermint/vm/actor_interface" }
 fendermint_vm_message = { path = "../../fendermint/vm/message" }
 fendermint_actor_blobs_shared = { path = "../../fendermint/actors/blobs/shared" }
+fendermint_actor_bucket = { path = "../../fendermint/actors/bucket" }
 fendermint_app = { path = "../../fendermint/app" }
 fendermint_crypto = { path = "../../fendermint/crypto" }
 ipc_ipld_resolver = { path = "../../ipld/resolver" }
 contracts-artifacts = { path = "../../contracts-artifacts" }
 ipc_actors_abis = { path = "../../contract-bindings" }
+walkdir = "2.4"
+dirs = "5.0"
+blake3 = "1.5"
+iroh-blobs = { workspace = true }

ipc/cli/Cargo.toml

@@ -0,0 +1,254 @@
+// Copyright 2022-2024 Protocol Labs
+// SPDX-License-Identifier: MIT
+
+//! Bucket operations for on-chain storage management
+//!
+//! This module provides functions to interact with bucket smart contracts.
+
+use anyhow::{anyhow, Context, Result};
+use fendermint_actor_bucket::{
+    AddParams, DeleteParams, GetParams, ListObjectsReturn, ListParams, Object, UpdateObjectMetadataParams,
+    Method as BucketMethod,
+};
+use fendermint_actor_blobs_shared::bytes::B256;
+use fendermint_rpc::{message::GasParams, tx::{BoundClient, TxClient, TxCommit}, QueryClient};
+use fendermint_vm_message::query::FvmQueryHeight;
+use fvm_ipld_encoding::RawBytes;
+use fvm_shared::{address::Address, econ::TokenAmount};
+use std::collections::HashMap;
+
+/// Default gas parameters for bucket transactions
+fn default_gas_params() -> GasParams {
+    GasParams {
+        gas_limit: 10_000_000_000,
+        gas_fee_cap: TokenAmount::from_atto(100),
+        gas_premium: TokenAmount::from_atto(100),
+    }
+}
+
+/// Add an object to a bucket
+///
+/// This registers an object's metadata on-chain after the blob has been uploaded
+/// to the gateway and distributed to storage nodes.
+pub async fn add_object<C>(
+    client: &mut C,
+    bucket_address: Address,
+    source: B256,
+    key: String,
+    hash: B256,
+    recovery_hash: B256,
+    size: u64,
+    metadata: HashMap<String, String>,
+    data_shards: u16,
+    parity_shards: u16,
+) -> Result<()>
+where
+    C: BoundClient + TxClient<TxCommit> + Send + Sync,
+{
+    let params = AddParams {
+        source,
+        key: key.into_bytes(),
+        hash,
+        recovery_hash,
+        size,
+        ttl: None, // Use default TTL
+        metadata,
+        overwrite: false,
+        data_shards,
+        parity_shards,
+    };
+
+    let params_bytes = RawBytes::serialize(params).context("Failed to serialize AddParams")?;
+
+    let res = TxClient::<TxCommit>::transaction(
+        client,
+        bucket_address,
+        BucketMethod::AddObject as u64,
+        params_bytes,
+        TokenAmount::zero(),
+        default_gas_params(),
+    )
+    .await
+    .context("Failed to send AddObject transaction")?;
+
+    if res.response.check_tx.code.is_err() {
+        return Err(anyhow!(
+            "AddObject check_tx failed: {}",
+            res.response.check_tx.log
+        ));
+    }
+
+    if res.response.deliver_tx.code.is_err() {
+        return Err(anyhow!(
+            "AddObject deliver_tx failed: {}",
+            res.response.deliver_tx.log
+        ));
+    }
+
+    Ok(())
+}
+
+/// Get an object from a bucket
+pub async fn get_object<C>(
+    client: &mut C,
+    bucket_address: Address,
+    key: String,
+) -> Result<Option<Object>>
+where
+    C: QueryClient + Send + Sync,
+{
+    let params = GetParams(key.into_bytes());
+    let gas_params = GasParams {
+        gas_limit: Default::default(),
+        gas_fee_cap: Default::default(),
+        gas_premium: Default::default(),
+    };
+
+    client
+        .os_get_call(
+            bucket_address,
+            params,
+            TokenAmount::default(),
+            gas_params,
+            FvmQueryHeight::default(),
+        )
+        .await
+        .context("Failed to query object")
+}
+
+/// List objects in a bucket
+pub async fn list_objects<C>(
+    client: &C,
+    bucket_address: Address,
+    prefix: Option<String>,
+    delimiter: Option<String>,
+    start_key: Option<String>,
+    limit: u64,
+) -> Result<ListObjectsReturn>
+where
+    C: QueryClient + Send + Sync,
+{
+    let params = ListParams {
+        prefix: prefix.unwrap_or_default().into_bytes(),
+        delimiter: delimiter.unwrap_or_default().into_bytes(),
+        start_key: start_key.map(|s| s.into_bytes()),
+        limit,
+    };
+
+    let params_bytes = RawBytes::serialize(params).context("Failed to serialize ListParams")?;
+
+    let msg = fvm_shared::message::Message {
+        version: Default::default(),
+        from: fendermint_vm_actor_interface::system::SYSTEM_ACTOR_ADDR,
+        to: bucket_address,
+        sequence: 0,
+        value: TokenAmount::zero(),
+        method_num: BucketMethod::ListObjects as u64,
+        params: params_bytes,
+        gas_limit: 10_000_000_000,
+        gas_fee_cap: TokenAmount::zero(),
+        gas_premium: TokenAmount::zero(),
+    };
+
+    let response = client
+        .call(msg, FvmQueryHeight::default())
+        .await
+        .context("Failed to execute ListObjects call")?;
+
+    if response.value.code.is_err() {
+        return Err(anyhow!("ListObjects query failed: {}", response.value.info));
+    }
+
+    let return_data = fendermint_rpc::response::decode_data(&response.value.data)
+        .context("Failed to decode response data")?;
+
+    let result = fvm_ipld_encoding::from_slice::<ListObjectsReturn>(&return_data)
+        .context("Failed to decode ListObjects response")?;
+
+    Ok(result)
+}
+
+/// Delete an object from a bucket
+pub async fn delete_object<C>(
+    client: &mut C,
+    bucket_address: Address,
+    key: String,
+) -> Result<()>
+where
+    C: BoundClient + TxClient<TxCommit> + Send + Sync,
+{
+    let params = DeleteParams(key.into_bytes());
+    let params_bytes = RawBytes::serialize(params).context("Failed to serialize DeleteParams")?;
+
+    let res = TxClient::<TxCommit>::transaction(
+        client,
+        bucket_address,
+        BucketMethod::DeleteObject as u64,
+        params_bytes,
+        TokenAmount::zero(),
+        default_gas_params(),
+    )
+    .await
+    .context("Failed to send DeleteObject transaction")?;
+
+    if res.response.check_tx.code.is_err() {
+        return Err(anyhow!(
+            "DeleteObject check_tx failed: {}",
+            res.response.check_tx.log
+        ));
+    }
+
+    if res.response.deliver_tx.code.is_err() {
+        return Err(anyhow!(
+            "DeleteObject deliver_tx failed: {}",
+            res.response.deliver_tx.log
+        ));
+    }
+
+    Ok(())
+}
+
+/// Update object metadata
+pub async fn update_object_metadata<C>(
+    client: &mut C,
+    bucket_address: Address,
+    key: String,
+    metadata: HashMap<String, Option<String>>,
+) -> Result<()>
+where
+    C: BoundClient + TxClient<TxCommit> + Send + Sync,
+{
+    let params = UpdateObjectMetadataParams {
+        key: key.into_bytes(),
+        metadata,
+    };
+    let params_bytes =
+        RawBytes::serialize(params).context("Failed to serialize UpdateObjectMetadataParams")?;
+
+    let res = TxClient::<TxCommit>::transaction(
+        client,
+        bucket_address,
+        BucketMethod::UpdateObjectMetadata as u64,
+        params_bytes,
+        TokenAmount::zero(),
+        default_gas_params(),
+    )
+    .await
+    .context("Failed to send UpdateObjectMetadata transaction")?;
+
+    if res.response.check_tx.code.is_err() {
+        return Err(anyhow!(
+            "UpdateObjectMetadata check_tx failed: {}",
+            res.response.check_tx.log
+        ));
+    }
+
+    if res.response.deliver_tx.code.is_err() {
+        return Err(anyhow!(
+            "UpdateObjectMetadata deliver_tx failed: {}",
+            res.response.deliver_tx.log
+        ));
+    }
+
+    Ok(())
+}

ipc/cli/src/commands/storage/bucket.rs

@@ -0,0 +1,78 @@
+// Copyright 2022-2024 Protocol Labs
+// SPDX-License-Identifier: MIT
+
+//! Cat command for displaying file contents from storage
+
+use anyhow::{anyhow, Context, Result};
+use clap::Args;
+use std::io::{self, Write};
+use std::path::PathBuf;
+
+use crate::commands::storage::{client::GatewayClient, config::StorageConfig, path};
+use crate::{CommandLineHandler, GlobalArguments};
+
+#[derive(Debug, Args)]
+pub struct CatArgs {
+    /// Storage path (ipc://bucket_address/path/to/file)
+    #[arg(value_name = "PATH")]
+    pub path: String,
+
+    /// Gateway URL (overrides config and env var)
+    #[arg(long)]
+    pub gateway: Option<String>,
+
+    /// Storage config file
+    #[arg(long)]
+    pub config: Option<PathBuf>,
+}
+
+pub struct CatStorage;
+
+impl CommandLineHandler for CatStorage {
+    type Arguments = CatArgs;
+
+    async fn handle(_global: &GlobalArguments, args: &Self::Arguments) -> Result<()> {
+        let storage_path = path::StoragePath::parse(&args.path)?;
+
+        if storage_path.is_bucket_root() {
+            return Err(anyhow!("Path must include a file key, not just bucket address"));
+        }
+
+        // Load config
+        let config_path = args.config.clone().unwrap_or_else(|| {
+            dirs::home_dir()
+                .unwrap()
+                .join(".ipc")
+                .join("storage_default.yaml")
+        });
+
+        let mut config = if config_path.exists() {
+            StorageConfig::load(&config_path)?
+        } else {
+            return Err(anyhow!(
+                "Storage config not found at {}. Run 'ipc-cli storage init' first.",
+                config_path.display()
+            ));
+        };
+
+        // Get gateway URL
+        let gateway_url = config.get_gateway_url(
+            args.gateway.as_deref(),
+            Some(&config_path),
+            false, // not interactive (avoid prompting when piping)
+        )?;
+
+        // Download from gateway
+        let gateway_client = GatewayClient::new(gateway_url)?;
+        let data = gateway_client
+            .download_object(&storage_path.bucket_address, &storage_path.key, None)
+            .await
+            .context("Failed to download object from gateway")?;
+
+        // Write to stdout
+        io::stdout().write_all(&data)?;
+        io::stdout().flush()?;
+
+        Ok(())
+    }
+}