Skip to content
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,50 @@ pub struct Options {
)]
pub hot_tier_storage_path: Option<PathBuf>,

#[arg(
long = "hot-tier-download-chunk-size",
env = "P_HOT_TIER_DOWNLOAD_CHUNK_SIZE",
value_parser = clap::value_parser!(u64).range(5242880..),
default_value = "8388608",
help = "Chunk size in bytes for parallel hot tier downloads (default 8 MiB)"
)]
pub hot_tier_download_chunk_size: u64,

#[arg(
long = "hot-tier-download-concurrency",
env = "P_HOT_TIER_DOWNLOAD_CONCURRENCY",
value_parser = clap::value_parser!(u64).range(1..),
default_value = "16",
help = "Number of concurrent range requests per hot tier download"
)]
pub hot_tier_download_concurrency: u64,
Comment thread
parmesant marked this conversation as resolved.

#[arg(
long = "hot-tier-files-per-stream-concurrency",
env = "P_HOT_TIER_FILES_PER_STREAM_CONCURRENCY",
default_value = "4",
help = "Number of concurrent parquet file downloads per stream during hot tier sync"
)]
pub hot_tier_files_per_stream_concurrency: usize,
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated

#[arg(
long = "hot-tier-latest-minutes",
env = "P_HOT_TIER_LATEST_MINUTES",
value_parser = clap::value_parser!(u64).range(1..),
default_value = "10",
help = "Files whose timestamp is within the last N minutes are 'latest'; rest are 'historic'."
)]
pub hot_tier_latest_minutes: u64,

#[arg(
long = "hot-tier-historic-sync-minutes",
env = "P_HOT_TIER_HISTORIC_SYNC_MINUTES",
value_parser = clap::value_parser!(u32).range(1..),
default_value = "5",
help = "Interval (minutes) at which the historic hot-tier sync runs."
)]
pub hot_tier_historic_sync_minutes: u32,
Comment thread
coderabbitai[bot] marked this conversation as resolved.

//TODO: remove this when smart cache is implemented
#[arg(
long = "index-storage-path",
Expand Down
4 changes: 3 additions & 1 deletion src/handlers/http/logstream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,9 @@ pub async fn put_stream_hot_tier(
.metastore
.put_stream_json(&stream_metadata, &stream_name, &tenant_id)
.await?;

hot_tier_manager
.spawn_stream_tasks(stream_name.clone(), tenant_id.clone())
.await;
Ok((
format!("hot tier set for stream {stream_name}"),
StatusCode::OK,
Expand Down
Loading
Loading