From 4ff1ce97679bf53873f1e8d4c33a17e8b46922ec Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Thu, 22 Feb 2024 15:04:48 +0200 Subject: [PATCH 01/60] Disaggregated write-ahead log for libSQL --- Cargo.lock | 36 ++ Cargo.toml | 2 + libsql-server/Cargo.toml | 1 + .../src/namespace/replication_wal.rs | 3 +- .../primary/replication_logger_wal.rs | 11 +- libsql-storage-server/Cargo.toml | 25 + libsql-storage-server/build.rs | 6 + libsql-storage-server/src/lib.rs | 1 + libsql-storage-server/src/main.rs | 119 ++++ libsql-storage-server/src/version.rs | 17 + libsql-storage/Cargo.toml | 19 + libsql-storage/proto/storage.proto | 47 ++ libsql-storage/src/generated/storage.rs | 549 ++++++++++++++++++ libsql-storage/src/lib.rs | 255 ++++++++ libsql-storage/tests/bootstrap.rs | 33 ++ 15 files changed, 1119 insertions(+), 5 deletions(-) create mode 100644 libsql-storage-server/Cargo.toml create mode 100644 libsql-storage-server/build.rs create mode 100644 libsql-storage-server/src/lib.rs create mode 100644 libsql-storage-server/src/main.rs create mode 100644 libsql-storage-server/src/version.rs create mode 100644 libsql-storage/Cargo.toml create mode 100644 libsql-storage/proto/storage.proto create mode 100644 libsql-storage/src/generated/storage.rs create mode 100644 libsql-storage/src/lib.rs create mode 100644 libsql-storage/tests/bootstrap.rs diff --git a/Cargo.lock b/Cargo.lock index bb12571684..9f957c112a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2808,6 +2808,7 @@ dependencies = [ "libsql-client", "libsql-rusqlite", "libsql-sqlite3-parser", + "libsql-storage", "libsql-sys", "libsql_replication", "md-5", @@ -2887,6 +2888,35 @@ dependencies = [ "uncased", ] +[[package]] +name = "libsql-storage" +version = "0.2.0" +dependencies = [ + "libsql-sys", + "prost", + "prost-build", + "sieve-cache", + "tokio", + "tonic", + "tonic-build", + "tracing", +] + +[[package]] +name = "libsql-storage-server" +version = "0.23.0" +dependencies = [ + "anyhow", + "bytes", + "clap 4.4.18", + "libsql-storage", + "tokio", + "tonic", + "tracing", + "tracing-subscriber", + "vergen", +] + [[package]] name = "libsql-sys" version = "0.3.0" @@ -4544,6 +4574,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "sieve-cache" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bf3a9dccf2c079bf1465d449a485c85b36443caf765f2f127bfec28b180f75" + [[package]] name = "signal-hook-registry" version = "1.4.1" diff --git a/Cargo.toml b/Cargo.toml index 1d34714c58..b0d5d8123e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,8 @@ members = [ "libsql-sys", "libsql-shell", "libsql-server", + "libsql-storage", + "libsql-storage-server", "bottomless", "bottomless-cli", "libsql-replication", diff --git a/libsql-server/Cargo.toml b/libsql-server/Cargo.toml index 6710cb594b..219d2b84d0 100644 --- a/libsql-server/Cargo.toml +++ b/libsql-server/Cargo.toml @@ -37,6 +37,7 @@ itertools = "0.10.5" jsonwebtoken = "8.2.0" libsql = { path = "../libsql/", optional = true } libsql_replication = { path = "../libsql-replication" } +libsql-storage = { path = "../libsql-storage" } metrics = "0.21.1" metrics-util = "0.15" metrics-exporter-prometheus = "0.12.2" diff --git a/libsql-server/src/namespace/replication_wal.rs b/libsql-server/src/namespace/replication_wal.rs index cae6aff915..2e4d4d1be8 100644 --- a/libsql-server/src/namespace/replication_wal.rs +++ b/libsql-server/src/namespace/replication_wal.rs @@ -17,8 +17,9 @@ pub fn make_replication_wal( bottomless: Option, logger: Arc, ) -> ReplicationWalManager { + let wal_manager = libsql_storage::DurableWalManager::new(); WalWrapper::new( bottomless.map(|b| BottomlessWalWrapper::new(Arc::new(std::sync::Mutex::new(Some(b))))), - ReplicationLoggerWalManager::new(logger), + ReplicationLoggerWalManager::new(wal_manager, logger), ) } diff --git a/libsql-server/src/replication/primary/replication_logger_wal.rs b/libsql-server/src/replication/primary/replication_logger_wal.rs index 8e22f76e69..7b7bb0074e 100644 --- a/libsql-server/src/replication/primary/replication_logger_wal.rs +++ b/libsql-server/src/replication/primary/replication_logger_wal.rs @@ -19,14 +19,17 @@ use super::logger::WalPage; #[derive(Clone)] pub struct ReplicationLoggerWalManager { - sqlite_wal_manager: Sqlite3WalManager, + sqlite_wal_manager: libsql_storage::DurableWalManager, logger: Arc, } impl ReplicationLoggerWalManager { - pub fn new(logger: Arc) -> Self { + pub fn new( + sqlite_wal_manager: libsql_storage::DurableWalManager, + logger: Arc, + ) -> Self { Self { - sqlite_wal_manager: Sqlite3WalManager::new(), + sqlite_wal_manager, logger, } } @@ -89,7 +92,7 @@ impl WalManager for ReplicationLoggerWalManager { } pub struct ReplicationLoggerWal { - inner: Sqlite3Wal, + inner: libsql_storage::DurableWal, buffer: Vec, logger: Arc, } diff --git a/libsql-storage-server/Cargo.toml b/libsql-storage-server/Cargo.toml new file mode 100644 index 0000000000..e480c414f3 --- /dev/null +++ b/libsql-storage-server/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "libsql-storage-server" +version = "0.23.0" +edition = "2021" +default-run = "libsql-storage-server" + +[[bin]] +name = "libsql-storage-server" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.66" +bytes = "1.5.0" +clap = { version = "4.0.23", features = [ "derive", "env", "string" ] } +libsql-storage = { path = "../libsql-storage" } +tokio = { version = "1.22.2", features = ["rt-multi-thread", "net", "io-std", "io-util", "time", "macros", "sync", "fs", "signal"] } +tonic = { version = "0.10.0", features = ["tls"] } +tracing = "0.1.37" +tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } + +[dev-dependencies] + +[build-dependencies] +vergen = { version = "8", features = ["build", "git", "gitcl"] } + diff --git a/libsql-storage-server/build.rs b/libsql-storage-server/build.rs new file mode 100644 index 0000000000..e8ec8ff95d --- /dev/null +++ b/libsql-storage-server/build.rs @@ -0,0 +1,6 @@ +use vergen::EmitBuilder; + +fn main() -> Result<(), Box> { + EmitBuilder::builder().git_sha(false).all_build().emit()?; + Ok(()) +} diff --git a/libsql-storage-server/src/lib.rs b/libsql-storage-server/src/lib.rs new file mode 100644 index 0000000000..a6db76ad4b --- /dev/null +++ b/libsql-storage-server/src/lib.rs @@ -0,0 +1 @@ +pub mod version; diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs new file mode 100644 index 0000000000..1edf5614ee --- /dev/null +++ b/libsql-storage-server/src/main.rs @@ -0,0 +1,119 @@ +use anyhow::Result; +use clap::Parser; +use libsql_storage::rpc::storage_server::{Storage, StorageServer}; +use libsql_storage::rpc::{ + DbSizeReq, DbSizeResp, FindFrameReq, FindFrameResp, InsertFramesReq, InsertFramesResp, + ReadFrameReq, ReadFrameResp, +}; +use libsql_storage_server::version::Version; +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::sync::atomic::AtomicU32; +use std::sync::{Arc, Mutex}; +use tonic::{transport::Server, Response}; +use tracing::trace; + +/// libSQL storage server +#[derive(Debug, Parser)] +#[command(name = "libsql-storage-server")] +#[command(about = "libSQL storage server", version = Version::default(), long_about = None)] +struct Cli { + /// The address and port the storage RPC protocol listens to. Example: `127.0.0.1:5002`. + #[clap( + long, + env = "LIBSQL_STORAGE_LISTEN_ADDR", + default_value = "127.0.0.1:5002" + )] + listen_addr: SocketAddr, +} + +#[derive(Default)] +struct Service { + pages: Arc>>, + db_size: AtomicU32, +} + +impl Service { + pub fn new() -> Self { + Self::default() + } +} + +#[tonic::async_trait] +impl Storage for Service { + async fn insert_frames( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + trace!("insert_frames()"); + let mut num_frames = 0; + for frame in request.into_inner().frames { + let mut pages = self.pages.lock().unwrap(); + trace!("inserting frame for page {}", frame.page_no); + pages.insert(frame.page_no, frame.data.into()); + num_frames += 1; + self.db_size + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + Ok(Response::new(InsertFramesResp { num_frames })) + } + + async fn find_frame( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let page_no = request.into_inner().page_no; + trace!("find_frame(page_no={})", page_no); + let pages = self.pages.lock().unwrap(); + if pages.contains_key(&page_no) { + // We have 1:1 mapping between frames and pages to cheat a bit. + Ok(Response::new(FindFrameResp { + frame_no: Some(page_no), + })) + } else { + Ok(Response::new(FindFrameResp { frame_no: None })) + } + } + + async fn read_frame( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let frame_no = request.into_inner().frame_no; + trace!("read_frame(frame_no={})", frame_no); + let pages = self.pages.lock().unwrap(); + if let Some(data) = pages.get(&frame_no) { + Ok(Response::new(ReadFrameResp { + frame: Some(data.clone().into()), + })) + } else { + Ok(Response::new(ReadFrameResp { frame: None })) + } + } + + async fn db_size( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let size = self.db_size.load(std::sync::atomic::Ordering::SeqCst) as u64; + Ok(Response::new(DbSizeResp { size })) + } +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt::init(); + + let args = Cli::parse(); + + let service = Service::default(); + + println!("Starting libSQL storage server on {}", args.listen_addr); + + Server::builder() + .add_service(StorageServer::new(service)) + .serve(args.listen_addr) + .await?; + + Ok(()) +} diff --git a/libsql-storage-server/src/version.rs b/libsql-storage-server/src/version.rs new file mode 100644 index 0000000000..4dc139ab01 --- /dev/null +++ b/libsql-storage-server/src/version.rs @@ -0,0 +1,17 @@ +use clap::builder::{IntoResettable, Str}; + +#[derive(Default)] +pub struct Version; + +impl IntoResettable for Version { + fn into_resettable(self) -> clap::builder::Resettable { + version().into_resettable() + } +} + +pub fn version() -> String { + let pkg_version = env!("CARGO_PKG_VERSION"); + let git_sha = env!("VERGEN_GIT_SHA"); + let build_date = env!("VERGEN_BUILD_DATE"); + format!("sqld {} ({} {})", pkg_version, &git_sha[..8], build_date) +} diff --git a/libsql-storage/Cargo.toml b/libsql-storage/Cargo.toml new file mode 100644 index 0000000000..14a4f01f52 --- /dev/null +++ b/libsql-storage/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "libsql-storage" +version = "0.2.0" +edition = "2021" +description = "libSQL WAL" +repository = "https://github.com/tursodatabase/libsql" +license = "MIT" + +[dependencies] +libsql-sys = { path = "../libsql-sys" } +prost = "0.12" +sieve-cache = "0.1.4" +tokio = { version = "1.22.2", features = ["rt-multi-thread", "net", "io-std", "io-util", "time", "macros", "sync", "fs"] } +tonic = { version = "0.10.0", features = ["tls"] } +tracing = { version = "0.1.37", default-features = false } + +[dev-dependencies] +prost-build = "0.12.0" +tonic-build = "0.10" diff --git a/libsql-storage/proto/storage.proto b/libsql-storage/proto/storage.proto new file mode 100644 index 0000000000..d785a413f5 --- /dev/null +++ b/libsql-storage/proto/storage.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package storage; + +message Frame { + uint64 page_no = 1; + bytes data = 2; +} + +message InsertFramesReq { + repeated Frame frames = 1; +} + +message InsertFramesResp { + int32 num_frames = 1; +} + +message FindFrameReq { + uint64 page_no = 1; +} + +message FindFrameResp { + optional uint64 frame_no = 1; +} + +message ReadFrameReq { + uint64 frame_no = 1; +} + +message ReadFrameResp { + optional bytes frame = 1; +} + +message DbSizeReq { + +} + +message DbSizeResp { + uint64 size = 1; +} + +service Storage { + rpc InsertFrames(InsertFramesReq) returns (InsertFramesResp) {} + rpc FindFrame(FindFrameReq) returns (FindFrameResp) {} + rpc ReadFrame(ReadFrameReq) returns (ReadFrameResp) {} + rpc DbSize(DbSizeReq) returns (DbSizeResp) {} +} diff --git a/libsql-storage/src/generated/storage.rs b/libsql-storage/src/generated/storage.rs new file mode 100644 index 0000000000..edcdc31970 --- /dev/null +++ b/libsql-storage/src/generated/storage.rs @@ -0,0 +1,549 @@ +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Frame { + #[prost(uint64, tag = "1")] + pub page_no: u64, + #[prost(bytes = "vec", tag = "2")] + pub data: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct InsertFramesReq { + #[prost(message, repeated, tag = "1")] + pub frames: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct InsertFramesResp { + #[prost(int32, tag = "1")] + pub num_frames: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FindFrameReq { + #[prost(uint64, tag = "1")] + pub page_no: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FindFrameResp { + #[prost(uint64, optional, tag = "1")] + pub frame_no: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadFrameReq { + #[prost(uint64, tag = "1")] + pub frame_no: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadFrameResp { + #[prost(bytes = "vec", optional, tag = "1")] + pub frame: ::core::option::Option<::prost::alloc::vec::Vec>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DbSizeReq {} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DbSizeResp { + #[prost(uint64, tag = "1")] + pub size: u64, +} +/// Generated client implementations. +pub mod storage_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct StorageClient { + inner: tonic::client::Grpc, + } + impl StorageClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl StorageClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> StorageClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + Send + Sync, + { + StorageClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn insert_frames( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/storage.Storage/InsertFrames", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("storage.Storage", "InsertFrames")); + self.inner.unary(req, path, codec).await + } + pub async fn find_frame( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/storage.Storage/FindFrame", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("storage.Storage", "FindFrame")); + self.inner.unary(req, path, codec).await + } + pub async fn read_frame( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/storage.Storage/ReadFrame", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("storage.Storage", "ReadFrame")); + self.inner.unary(req, path, codec).await + } + pub async fn db_size( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/storage.Storage/DbSize"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("storage.Storage", "DbSize")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod storage_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with StorageServer. + #[async_trait] + pub trait Storage: Send + Sync + 'static { + async fn insert_frames( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + async fn find_frame( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn read_frame( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn db_size( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct StorageServer { + inner: _Inner, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + struct _Inner(Arc); + impl StorageServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + let inner = _Inner(inner); + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for StorageServer + where + T: Storage, + B: Body + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/storage.Storage/InsertFrames" => { + #[allow(non_camel_case_types)] + struct InsertFramesSvc(pub Arc); + impl tonic::server::UnaryService + for InsertFramesSvc { + type Response = super::InsertFramesResp; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::insert_frames(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = InsertFramesSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/storage.Storage/FindFrame" => { + #[allow(non_camel_case_types)] + struct FindFrameSvc(pub Arc); + impl tonic::server::UnaryService + for FindFrameSvc { + type Response = super::FindFrameResp; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::find_frame(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = FindFrameSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/storage.Storage/ReadFrame" => { + #[allow(non_camel_case_types)] + struct ReadFrameSvc(pub Arc); + impl tonic::server::UnaryService + for ReadFrameSvc { + type Response = super::ReadFrameResp; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::read_frame(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = ReadFrameSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/storage.Storage/DbSize" => { + #[allow(non_camel_case_types)] + struct DbSizeSvc(pub Arc); + impl tonic::server::UnaryService + for DbSizeSvc { + type Response = super::DbSizeResp; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::db_size(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = DbSizeSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + Ok( + http::Response::builder() + .status(200) + .header("grpc-status", "12") + .header("content-type", "application/grpc") + .body(empty_body()) + .unwrap(), + ) + }) + } + } + } + } + impl Clone for StorageServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::server::NamedService for StorageServer { + const NAME: &'static str = "storage.Storage"; + } +} diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs new file mode 100644 index 0000000000..4d4d070309 --- /dev/null +++ b/libsql-storage/src/lib.rs @@ -0,0 +1,255 @@ +use libsql_sys::wal::{Result, Vfs, Wal, WalManager}; +use sieve_cache::SieveCache; +use tonic::transport::Channel; +use tracing::trace; + +pub mod rpc { + #![allow(clippy::all)] + include!("generated/storage.rs"); +} + +use rpc::storage_client::StorageClient; + +#[derive(Clone)] +pub struct DurableWalManager {} + +impl DurableWalManager { + pub fn new() -> Self { + Self {} + } +} + +impl WalManager for DurableWalManager { + type Wal = DurableWal; + + fn use_shared_memory(&self) -> bool { + trace!("DurableWalManager::use_shared_memory()"); + false + } + + fn open( + &self, + vfs: &mut Vfs, + file: &mut libsql_sys::wal::Sqlite3File, + no_shm_mode: std::ffi::c_int, + max_log_size: i64, + db_path: &std::ffi::CStr, + ) -> Result { + let db_path = db_path.to_str().unwrap(); + trace!("DurableWalManager::open(db_path: {})", db_path); + Ok(DurableWal::new()) + } + + fn close( + &self, + wal: &mut Self::Wal, + db: &mut libsql_sys::wal::Sqlite3Db, + sync_flags: std::ffi::c_int, + scratch: Option<&mut [u8]>, + ) -> Result<()> { + trace!("DurableWalManager::close()"); + Ok(()) + } + + fn destroy_log(&self, vfs: &mut Vfs, db_path: &std::ffi::CStr) -> Result<()> { + trace!("DurableWalManager::destroy_log()"); + // TODO: implement + Ok(()) + } + + fn log_exists(&self, vfs: &mut Vfs, db_path: &std::ffi::CStr) -> Result { + trace!("DurableWalManager::log_exists()"); + // TODO: implement + Ok(false) + } + + fn destroy(self) + where + Self: Sized, + { + trace!("DurableWalManager::destroy()"); + } +} + +pub struct DurableWal { + client: StorageClient, + page_frames: SieveCache, + db_size: u32, +} + +impl DurableWal { + fn new() -> Self { + let rt = tokio::runtime::Handle::current(); + let client = StorageClient::connect("http://127.0.0.1:5002"); + let mut client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); + + let req = rpc::DbSizeReq {}; + let resp = client.db_size(req); + let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); + let db_size = resp.into_inner().size as u32; + + let page_frames = SieveCache::new(1000).unwrap(); + + Self { + client, + page_frames, + db_size, + } + } +} + +impl Wal for DurableWal { + fn limit(&mut self, size: i64) { + todo!() + } + + fn begin_read_txn(&mut self) -> Result { + trace!("DurableWal::begin_read_txn()"); + Ok(true) + } + + fn end_read_txn(&mut self) { + trace!("DurableWal::end_read_txn()"); + } + + fn find_frame( + &mut self, + page_no: std::num::NonZeroU32, + ) -> Result> { + trace!("DurableWal::find_frame(page_no: {:?})", page_no); + if let Some(frame_no) = self.page_frames.get(&page_no) { + return Ok(Some(*frame_no)); + } + let rt = tokio::runtime::Handle::current(); + let req = rpc::FindFrameReq { + page_no: page_no.get() as u64, + }; + let resp = self.client.find_frame(req); + let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); + let frame_no = resp + .into_inner() + .frame_no + .map(|page_no| std::num::NonZeroU32::new(page_no as u32)) + .flatten(); + if let Some(frame_no) = frame_no { + self.page_frames.insert(page_no, frame_no); + } + Ok(frame_no) + } + + fn read_frame(&mut self, frame_no: std::num::NonZeroU32, buffer: &mut [u8]) -> Result<()> { + trace!("DurableWal::read_frame(frame_no: {:?})", frame_no); + let rt = tokio::runtime::Handle::current(); + let frame_no = frame_no.get() as u64; + let req = rpc::ReadFrameReq { frame_no }; + let resp = self.client.read_frame(req); + let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); + let frame = resp.into_inner().frame.unwrap(); + buffer.copy_from_slice(&frame); + Ok(()) + } + + fn frame_page_no(&self, frame_no: std::num::NonZeroU32) -> Option { + todo!() + } + + fn db_size(&self) -> u32 { + trace!("DurableWal::db_size() => {}", self.db_size); + self.db_size + } + + fn begin_write_txn(&mut self) -> Result<()> { + trace!("DurableWal::begin_write_txn()"); + Ok(()) + } + + fn end_write_txn(&mut self) -> Result<()> { + trace!("DurableWal::end_write_txn()"); + Ok(()) + } + + fn undo(&mut self, handler: Option<&mut U>) -> Result<()> { + todo!() + } + + fn savepoint(&mut self, rollback_data: &mut [u32]) { + todo!() + } + + fn savepoint_undo(&mut self, rollback_data: &mut [u32]) -> Result<()> { + todo!() + } + + fn insert_frames( + &mut self, + page_size: std::ffi::c_int, + page_headers: &mut libsql_sys::wal::PageHeaders, + size_after: u32, + is_commit: bool, + sync_flags: std::ffi::c_int, + ) -> Result { + trace!("DurableWal::insert_frames(page_size: {}, size_after: {}, is_commit: {}, sync_flags: {})", page_size, size_after, is_commit, sync_flags); + let rt = tokio::runtime::Handle::current(); + let frames = page_headers + .iter() + .map(|header| { + let (page_no, frame) = header; + rpc::Frame { + page_no: page_no as u64, + data: frame.to_vec(), + } + }) + .collect(); + let req = rpc::InsertFramesReq { frames }; + let resp = self.client.insert_frames(req); + let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); + self.db_size = size_after; + Ok(resp.into_inner().num_frames as usize) + } + + fn checkpoint( + &mut self, + db: &mut libsql_sys::wal::Sqlite3Db, + mode: libsql_sys::wal::CheckpointMode, + busy_handler: Option<&mut dyn libsql_sys::wal::BusyHandler>, + sync_flags: u32, + // temporary scratch buffer + buf: &mut [u8], + checkpoint_cb: Option<&mut dyn libsql_sys::wal::CheckpointCallback>, + in_wal: Option<&mut i32>, + backfilled: Option<&mut i32>, + ) -> Result<()> { + todo!() + } + + fn exclusive_mode(&mut self, op: std::ffi::c_int) -> Result<()> { + trace!("DurableWal::exclusive_mode(op: {})", op); + Ok(()) + } + + fn uses_heap_memory(&self) -> bool { + trace!("DurableWal::uses_heap_memory()"); + false + } + + fn set_db(&mut self, db: &mut libsql_sys::wal::Sqlite3Db) { + todo!() + } + + fn callback(&self) -> i32 { + trace!("DurableWal::callback()"); + 0 + } + + fn frames_in_wal(&self) -> u32 { + todo!() + } + + fn backfilled(&self) -> u32 { + todo!() + } + + fn db_file(&self) -> &libsql_sys::wal::Sqlite3File { + todo!() + } +} diff --git a/libsql-storage/tests/bootstrap.rs b/libsql-storage/tests/bootstrap.rs new file mode 100644 index 0000000000..e259f0e3b3 --- /dev/null +++ b/libsql-storage/tests/bootstrap.rs @@ -0,0 +1,33 @@ +use std::{path::PathBuf, process::Command}; + +#[test] +fn bootstrap() { + let iface_files = &["proto/storage.proto"]; + let dirs = &["storage"]; + + let out_dir = PathBuf::from(std::env!("CARGO_MANIFEST_DIR")) + .join("src") + .join("generated"); + + let mut config = prost_build::Config::new(); + config.bytes([".wal_log"]); + + tonic_build::configure() + .build_client(true) + .build_server(true) + .build_transport(true) + .out_dir(&out_dir) + .type_attribute(".proxy", "#[derive(serde::Serialize, serde::Deserialize)]") + .compile_with_config(config, iface_files, dirs) + .unwrap(); + + let status = Command::new("git") + .arg("diff") + .arg("--exit-code") + .arg("--") + .arg(&out_dir) + .status() + .unwrap(); + + assert!(status.success(), "You should commit the protobuf files"); +} From caadc9ae57ff891c32a4a8e382908203ef0cccdf Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Fri, 29 Mar 2024 17:46:38 +0530 Subject: [PATCH 02/60] Move to frame store --- libsql-storage-server/src/main.rs | 66 ++++++++++++++++++++++++++----- libsql-storage/src/lib.rs | 5 ++- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 1edf5614ee..23cebb1571 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -7,11 +7,13 @@ use libsql_storage::rpc::{ }; use libsql_storage_server::version::Version; use std::collections::BTreeMap; +use std::iter::Map; use std::net::SocketAddr; use std::sync::atomic::AtomicU32; use std::sync::{Arc, Mutex}; use tonic::{transport::Server, Response}; use tracing::trace; +use tracing_subscriber::{EnvFilter, FmtSubscriber}; /// libSQL storage server #[derive(Debug, Parser)] @@ -27,9 +29,48 @@ struct Cli { listen_addr: SocketAddr, } +#[derive(Default)] +struct FrameStore { + // contains a frame data, key is the frame number + frames: BTreeMap, + // pages map contains the page number as a key and the list of frames for the page as a value + pages: BTreeMap>, + max_frame_no: u64, +} + +impl FrameStore { + pub fn new() -> Self { + Self::default() + } + + // inserts a new frame for the page number and returns the new frame value + pub fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64 { + let frame_no = self.max_frame_no + 1; + self.max_frame_no = frame_no; + self.frames.insert(frame_no, frame); + self.pages + .entry(page_no) + .or_insert_with(Vec::new) + .push(frame_no); + frame_no + } + + pub fn read_frame(&self, frame_no: u64) -> Option<&bytes::Bytes> { + self.frames.get(&frame_no) + } + + // given a page number, return the maximum frame for the page + pub fn find_frame(&self, page_no: u64) -> Option { + self.pages + .get(&page_no) + .map(|frames| *frames.last().unwrap()) + } +} + #[derive(Default)] struct Service { pages: Arc>>, + store: Arc>, db_size: AtomicU32, } @@ -47,10 +88,10 @@ impl Storage for Service { ) -> Result, tonic::Status> { trace!("insert_frames()"); let mut num_frames = 0; + let mut store = self.store.lock().unwrap(); for frame in request.into_inner().frames { - let mut pages = self.pages.lock().unwrap(); trace!("inserting frame for page {}", frame.page_no); - pages.insert(frame.page_no, frame.data.into()); + store.insert_frame(frame.page_no, frame.data.into()); num_frames += 1; self.db_size .fetch_add(1, std::sync::atomic::Ordering::SeqCst); @@ -64,11 +105,9 @@ impl Storage for Service { ) -> Result, tonic::Status> { let page_no = request.into_inner().page_no; trace!("find_frame(page_no={})", page_no); - let pages = self.pages.lock().unwrap(); - if pages.contains_key(&page_no) { - // We have 1:1 mapping between frames and pages to cheat a bit. + if let Some(frame_no) = self.store.lock().unwrap().find_frame(page_no) { Ok(Response::new(FindFrameResp { - frame_no: Some(page_no), + frame_no: Some(frame_no), })) } else { Ok(Response::new(FindFrameResp { frame_no: None })) @@ -81,8 +120,7 @@ impl Storage for Service { ) -> Result, tonic::Status> { let frame_no = request.into_inner().frame_no; trace!("read_frame(frame_no={})", frame_no); - let pages = self.pages.lock().unwrap(); - if let Some(data) = pages.get(&frame_no) { + if let Some(data) = self.store.lock().unwrap().read_frame(frame_no) { Ok(Response::new(ReadFrameResp { frame: Some(data.clone().into()), })) @@ -102,14 +140,22 @@ impl Storage for Service { #[tokio::main] async fn main() -> Result<()> { - tracing_subscriber::fmt::init(); + let filter = EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new("libsql_storage_server=trace")); + tracing::subscriber::set_global_default( + FmtSubscriber::builder().with_env_filter(filter).finish(), + ) + .expect("setting default subscriber failed"); let args = Cli::parse(); let service = Service::default(); println!("Starting libSQL storage server on {}", args.listen_addr); - + trace!( + "(trace) Starting libSQL storage server on {}", + args.listen_addr + ); Server::builder() .add_service(StorageServer::new(service)) .serve(args.listen_addr) diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 4d4d070309..168e095fb9 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -219,7 +219,8 @@ impl Wal for DurableWal { in_wal: Option<&mut i32>, backfilled: Option<&mut i32>, ) -> Result<()> { - todo!() + // checkpoint is a no op + Ok(()) } fn exclusive_mode(&mut self, op: std::ffi::c_int) -> Result<()> { @@ -242,7 +243,7 @@ impl Wal for DurableWal { } fn frames_in_wal(&self) -> u32 { - todo!() + 0 } fn backfilled(&self) -> u32 { From ef933234b289fcee9bfac50dccbca5b49c30026b Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Fri, 29 Mar 2024 21:08:30 +0530 Subject: [PATCH 03/60] Add frame count and framepage req --- libsql-storage/proto/storage.proto | 18 +++ libsql-storage/src/generated/storage.rs | 170 ++++++++++++++++++++++++ 2 files changed, 188 insertions(+) diff --git a/libsql-storage/proto/storage.proto b/libsql-storage/proto/storage.proto index d785a413f5..72496d4b14 100644 --- a/libsql-storage/proto/storage.proto +++ b/libsql-storage/proto/storage.proto @@ -39,9 +39,27 @@ message DbSizeResp { uint64 size = 1; } +message FramesInWALReq { + +} + +message FramesInWALResp { + uint32 count = 1; +} + +message FramePageNumReq { + uint64 frame_no = 1; +} + +message FramePageNumResp { + uint64 page_no = 1; +} + service Storage { rpc InsertFrames(InsertFramesReq) returns (InsertFramesResp) {} rpc FindFrame(FindFrameReq) returns (FindFrameResp) {} rpc ReadFrame(ReadFrameReq) returns (ReadFrameResp) {} rpc DbSize(DbSizeReq) returns (DbSizeResp) {} + rpc FramesInWAL(FramesInWALReq) returns (FramesInWALResp) {} + rpc FramePageNum(FramePageNumReq) returns (FramePageNumResp) {} } diff --git a/libsql-storage/src/generated/storage.rs b/libsql-storage/src/generated/storage.rs index edcdc31970..b4ce06ffa2 100644 --- a/libsql-storage/src/generated/storage.rs +++ b/libsql-storage/src/generated/storage.rs @@ -51,6 +51,27 @@ pub struct DbSizeResp { #[prost(uint64, tag = "1")] pub size: u64, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FramesInWalReq {} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FramesInWalResp { + #[prost(uint32, tag = "1")] + pub count: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FramePageNumReq { + #[prost(uint64, tag = "1")] + pub frame_no: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FramePageNumResp { + #[prost(uint64, tag = "1")] + pub page_no: u64, +} /// Generated client implementations. pub mod storage_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] @@ -222,6 +243,56 @@ pub mod storage_client { req.extensions_mut().insert(GrpcMethod::new("storage.Storage", "DbSize")); self.inner.unary(req, path, codec).await } + pub async fn frames_in_wal( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/storage.Storage/FramesInWAL", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("storage.Storage", "FramesInWAL")); + self.inner.unary(req, path, codec).await + } + pub async fn frame_page_num( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/storage.Storage/FramePageNum", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("storage.Storage", "FramePageNum")); + self.inner.unary(req, path, codec).await + } } } /// Generated server implementations. @@ -250,6 +321,17 @@ pub mod storage_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn frames_in_wal( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn frame_page_num( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; } #[derive(Debug)] pub struct StorageServer { @@ -506,6 +588,94 @@ pub mod storage_server { }; Box::pin(fut) } + "/storage.Storage/FramesInWAL" => { + #[allow(non_camel_case_types)] + struct FramesInWALSvc(pub Arc); + impl tonic::server::UnaryService + for FramesInWALSvc { + type Response = super::FramesInWalResp; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::frames_in_wal(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = FramesInWALSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/storage.Storage/FramePageNum" => { + #[allow(non_camel_case_types)] + struct FramePageNumSvc(pub Arc); + impl tonic::server::UnaryService + for FramePageNumSvc { + type Response = super::FramePageNumResp; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::frame_page_num(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = FramePageNumSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } _ => { Box::pin(async move { Ok( From 6ab7f76c44897f066cf01016d86178fe6ea58baa Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Fri, 29 Mar 2024 21:25:30 +0530 Subject: [PATCH 04/60] Add `frame_page_num` and `frames_in_wal` --- libsql-storage-server/src/main.rs | 60 ++++++++++++++++++++++++++----- libsql-storage/src/lib.rs | 20 ++++++++--- 2 files changed, 68 insertions(+), 12 deletions(-) diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 23cebb1571..da4a6400c9 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -2,8 +2,9 @@ use anyhow::Result; use clap::Parser; use libsql_storage::rpc::storage_server::{Storage, StorageServer}; use libsql_storage::rpc::{ - DbSizeReq, DbSizeResp, FindFrameReq, FindFrameResp, InsertFramesReq, InsertFramesResp, - ReadFrameReq, ReadFrameResp, + DbSizeReq, DbSizeResp, FindFrameReq, FindFrameResp, FramePageNumReq, FramePageNumResp, + FramesInWalReq, FramesInWalResp, InsertFramesReq, InsertFramesResp, ReadFrameReq, + ReadFrameResp, }; use libsql_storage_server::version::Version; use std::collections::BTreeMap; @@ -11,8 +12,8 @@ use std::iter::Map; use std::net::SocketAddr; use std::sync::atomic::AtomicU32; use std::sync::{Arc, Mutex}; -use tonic::{transport::Server, Response}; -use tracing::trace; +use tonic::{transport::Server, Request, Response, Status}; +use tracing::{error, trace}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; /// libSQL storage server @@ -29,10 +30,17 @@ struct Cli { listen_addr: SocketAddr, } +#[derive(Default)] + +struct FrameData { + page_no: u64, + data: bytes::Bytes, +} + #[derive(Default)] struct FrameStore { // contains a frame data, key is the frame number - frames: BTreeMap, + frames: BTreeMap, // pages map contains the page number as a key and the list of frames for the page as a value pages: BTreeMap>, max_frame_no: u64, @@ -47,7 +55,13 @@ impl FrameStore { pub fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64 { let frame_no = self.max_frame_no + 1; self.max_frame_no = frame_no; - self.frames.insert(frame_no, frame); + self.frames.insert( + frame_no, + FrameData { + page_no, + data: frame, + }, + ); self.pages .entry(page_no) .or_insert_with(Vec::new) @@ -56,7 +70,7 @@ impl FrameStore { } pub fn read_frame(&self, frame_no: u64) -> Option<&bytes::Bytes> { - self.frames.get(&frame_no) + self.frames.get(&frame_no).map(|frame| &frame.data) } // given a page number, return the maximum frame for the page @@ -65,11 +79,19 @@ impl FrameStore { .get(&page_no) .map(|frames| *frames.last().unwrap()) } + + // given a frame num, return the page number + pub fn frame_page_no(&self, frame_no: u64) -> Option { + self.frames.get(&frame_no).map(|frame| frame.page_no) + } + + pub fn frames_in_wal(&self) -> u64 { + self.max_frame_no + } } #[derive(Default)] struct Service { - pages: Arc>>, store: Arc>, db_size: AtomicU32, } @@ -136,6 +158,28 @@ impl Storage for Service { let size = self.db_size.load(std::sync::atomic::Ordering::SeqCst) as u64; Ok(Response::new(DbSizeResp { size })) } + + async fn frames_in_wal( + &self, + request: Request, + ) -> std::result::Result, Status> { + Ok(Response::new(FramesInWalResp { + count: self.store.lock().unwrap().frames_in_wal() as u32, + })) + } + + async fn frame_page_num( + &self, + request: Request, + ) -> std::result::Result, Status> { + let frame_no = request.into_inner().frame_no; + if let Some(page_no) = self.store.lock().unwrap().frame_page_no(frame_no) { + Ok(Response::new(FramePageNumResp { page_no })) + } else { + error!("frame_page_num() failed for frame_no={}", frame_no); + Ok(Response::new(FramePageNumResp { page_no: 0 })) + } + } } #[tokio::main] diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 168e095fb9..b01c8b877e 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -100,7 +100,7 @@ impl DurableWal { impl Wal for DurableWal { fn limit(&mut self, size: i64) { - todo!() + // no op, we go bottomless baby! } fn begin_read_txn(&mut self) -> Result { @@ -150,7 +150,14 @@ impl Wal for DurableWal { } fn frame_page_no(&self, frame_no: std::num::NonZeroU32) -> Option { - todo!() + trace!("DurableWal::frame_page_no(frame_no: {:?})", frame_no); + let rt = tokio::runtime::Handle::current(); + let frame_no = frame_no.get() as u64; + let req = rpc::FramePageNumReq { frame_no }; + let resp = self.client.frame_page_num(req); + let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); + let page_no = resp.into_inner().page_no; + std::num::NonZeroU32::new(page_no as u32) } fn db_size(&self) -> u32 { @@ -242,8 +249,13 @@ impl Wal for DurableWal { 0 } - fn frames_in_wal(&self) -> u32 { - 0 + fn frames_in_wal(&mut self) -> u32 { + trace!("DurableWal::frames_in_wal()"); + let rt = tokio::runtime::Handle::current(); + let req = rpc::FramesInWalReq {}; + let resp = self.client.frames_in_wal(req); + let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); + resp.into_inner().count } fn backfilled(&self) -> u32 { From 0376491812d9e98151e9df65a977d4b4131a4875 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Fri, 29 Mar 2024 21:41:07 +0530 Subject: [PATCH 05/60] Update WAL trait to use mut --- libsql-replication/src/injector/injector_wal.rs | 4 ++-- .../replication/primary/replication_logger_wal.rs | 4 ++-- libsql-storage/src/lib.rs | 2 +- libsql-sys/src/wal/either.rs | 4 ++-- libsql-sys/src/wal/mod.rs | 4 ++-- libsql-sys/src/wal/sqlite3_wal.rs | 4 ++-- libsql-sys/src/wal/wrapper.rs | 14 +++++++------- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/libsql-replication/src/injector/injector_wal.rs b/libsql-replication/src/injector/injector_wal.rs index 3f2da8aab9..3b4f82ddeb 100644 --- a/libsql-replication/src/injector/injector_wal.rs +++ b/libsql-replication/src/injector/injector_wal.rs @@ -218,7 +218,7 @@ impl Wal for InjectorWal { self.inner.callback() } - fn frames_in_wal(&self) -> u32 { + fn frames_in_wal(&mut self) -> u32 { self.inner.frames_in_wal() } @@ -230,7 +230,7 @@ impl Wal for InjectorWal { self.inner.backfilled() } - fn frame_page_no(&self, frame_no: NonZeroU32) -> Option { + fn frame_page_no(&mut self, frame_no: NonZeroU32) -> Option { self.inner.frame_page_no(frame_no) } } diff --git a/libsql-server/src/replication/primary/replication_logger_wal.rs b/libsql-server/src/replication/primary/replication_logger_wal.rs index 7b7bb0074e..4d6167cd86 100644 --- a/libsql-server/src/replication/primary/replication_logger_wal.rs +++ b/libsql-server/src/replication/primary/replication_logger_wal.rs @@ -268,7 +268,7 @@ impl Wal for ReplicationLoggerWal { self.inner.callback() } - fn frames_in_wal(&self) -> u32 { + fn frames_in_wal(&mut self) -> u32 { self.inner.frames_in_wal() } @@ -280,7 +280,7 @@ impl Wal for ReplicationLoggerWal { self.inner.backfilled() } - fn frame_page_no(&self, frame_no: NonZeroU32) -> Option { + fn frame_page_no(&mut self, frame_no: NonZeroU32) -> Option { self.inner.frame_page_no(frame_no) } } diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index b01c8b877e..00b9b10345 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -149,7 +149,7 @@ impl Wal for DurableWal { Ok(()) } - fn frame_page_no(&self, frame_no: std::num::NonZeroU32) -> Option { + fn frame_page_no(&mut self, frame_no: std::num::NonZeroU32) -> Option { trace!("DurableWal::frame_page_no(frame_no: {:?})", frame_no); let rt = tokio::runtime::Handle::current(); let frame_no = frame_no.get() as u64; diff --git a/libsql-sys/src/wal/either.rs b/libsql-sys/src/wal/either.rs index 683a9a8cd6..f0f8bda43f 100644 --- a/libsql-sys/src/wal/either.rs +++ b/libsql-sys/src/wal/either.rs @@ -176,7 +176,7 @@ where } } - fn frames_in_wal(&self) -> u32 { + fn frames_in_wal(&mut self) -> u32 { match self { Either::Left(l) => l.frames_in_wal(), Either::Right(r) => r.frames_in_wal(), @@ -197,7 +197,7 @@ where } } - fn frame_page_no(&self, frame_no: std::num::NonZeroU32) -> Option { + fn frame_page_no(&mut self, frame_no: std::num::NonZeroU32) -> Option { match self { Either::Left(l) => l.frame_page_no(frame_no), Either::Right(r) => r.frame_page_no(frame_no), diff --git a/libsql-sys/src/wal/mod.rs b/libsql-sys/src/wal/mod.rs index db8d11b38c..2a682bde43 100644 --- a/libsql-sys/src/wal/mod.rs +++ b/libsql-sys/src/wal/mod.rs @@ -190,7 +190,7 @@ pub trait Wal { fn find_frame(&mut self, page_no: NonZeroU32) -> Result>; /// reads frame `frame_no` into buffer. fn read_frame(&mut self, frame_no: NonZeroU32, buffer: &mut [u8]) -> Result<()>; - fn frame_page_no(&self, frame_no: NonZeroU32) -> Option; + fn frame_page_no(&mut self, frame_no: NonZeroU32) -> Option; fn db_size(&self) -> u32; @@ -237,7 +237,7 @@ pub trait Wal { /// the last call, then return 0. fn callback(&self) -> i32; - fn frames_in_wal(&self) -> u32; + fn frames_in_wal(&mut self) -> u32; fn backfilled(&self) -> u32; fn db_file(&self) -> &Sqlite3File; } diff --git a/libsql-sys/src/wal/sqlite3_wal.rs b/libsql-sys/src/wal/sqlite3_wal.rs index 6d68b81ad3..651f0c2eca 100644 --- a/libsql-sys/src/wal/sqlite3_wal.rs +++ b/libsql-sys/src/wal/sqlite3_wal.rs @@ -409,7 +409,7 @@ impl Wal for Sqlite3Wal { unsafe { (self.inner.methods.xCallback.unwrap())(self.inner.pData) } } - fn frames_in_wal(&self) -> u32 { + fn frames_in_wal(&mut self) -> u32 { unsafe { let wal = &*(self.inner.pData as *const sqlite3_wal); wal.hdr.mxFrame @@ -435,7 +435,7 @@ impl Wal for Sqlite3Wal { /// ported from wal.c /// returns the page_no corresponding to a given frame_no - fn frame_page_no(&self, frame_no: NonZeroU32) -> Option { + fn frame_page_no(&mut self, frame_no: NonZeroU32) -> Option { unsafe { let ptr = &mut (*(self.inner.pData as *mut sqlite3_wal)); NonZeroU32::new(libsql_ffi::sqlite3_wal_frame_page_no(ptr, frame_no.get())) diff --git a/libsql-sys/src/wal/wrapper.rs b/libsql-sys/src/wal/wrapper.rs index 08af672f00..758dc85b2d 100644 --- a/libsql-sys/src/wal/wrapper.rs +++ b/libsql-sys/src/wal/wrapper.rs @@ -199,8 +199,8 @@ where self.wrapper.callback(&self.wrapped) } - fn frames_in_wal(&self) -> u32 { - self.wrapper.frames_in_wal(&self.wrapped) + fn frames_in_wal(&mut self) -> u32 { + self.wrapper.frames_in_wal(&mut self.wrapped) } fn db_file(&self) -> &super::Sqlite3File { @@ -211,8 +211,8 @@ where self.wrapped.backfilled() } - fn frame_page_no(&self, frame_no: NonZeroU32) -> Option { - self.wrapper.frame_page_no(&self.wrapped, frame_no) + fn frame_page_no(&mut self, frame_no: NonZeroU32) -> Option { + self.wrapper.frame_page_no(&mut self.wrapped, frame_no) } } @@ -329,7 +329,7 @@ pub trait WrapWal { wrapped.callback() } - fn frames_in_wal(&self, wrapped: &W) -> u32 { + fn frames_in_wal(&self, wrapped: &mut W) -> u32 { wrapped.frames_in_wal() } @@ -356,7 +356,7 @@ pub trait WrapWal { manager.close(wrapped, db, sync_flags, scratch) } - fn frame_page_no(&self, wrapped: &W, frame_no: NonZeroU32) -> Option { + fn frame_page_no(&mut self, wrapped: &mut W, frame_no: NonZeroU32) -> Option { wrapped.frame_page_no(frame_no) } } @@ -542,7 +542,7 @@ impl, W: Wal> WrapWal for Option { } } - fn frames_in_wal(&self, wrapped: &W) -> u32 { + fn frames_in_wal(&self, wrapped: &mut W) -> u32 { match self { Some(t) => t.frames_in_wal(wrapped), None => wrapped.frames_in_wal(), From dcae758e8f84a620095e6346d9f7e1d21c4335a9 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Mon, 1 Apr 2024 13:11:00 +0530 Subject: [PATCH 06/60] Use cache to store the frames --- libsql-storage-server/src/main.rs | 9 +++++++-- libsql-storage/src/lib.rs | 18 ++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index da4a6400c9..fb1409a1c3 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -112,8 +112,11 @@ impl Storage for Service { let mut num_frames = 0; let mut store = self.store.lock().unwrap(); for frame in request.into_inner().frames { - trace!("inserting frame for page {}", frame.page_no); - store.insert_frame(frame.page_no, frame.data.into()); + trace!( + "inserted for page {} frame {}", + frame.page_no, + store.insert_frame(frame.page_no, frame.data.into()) + ); num_frames += 1; self.db_size .fetch_add(1, std::sync::atomic::Ordering::SeqCst); @@ -132,6 +135,7 @@ impl Storage for Service { frame_no: Some(frame_no), })) } else { + error!("find_frame() failed for page_no={}", page_no); Ok(Response::new(FindFrameResp { frame_no: None })) } } @@ -147,6 +151,7 @@ impl Storage for Service { frame: Some(data.clone().into()), })) } else { + error!("read_frame() failed for frame_no={}", frame_no); Ok(Response::new(ReadFrameResp { frame: None })) } } diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 00b9b10345..eef2b9eca5 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -73,7 +73,7 @@ impl WalManager for DurableWalManager { pub struct DurableWal { client: StorageClient, - page_frames: SieveCache, + page_frames: SieveCache>, db_size: u32, } @@ -117,9 +117,6 @@ impl Wal for DurableWal { page_no: std::num::NonZeroU32, ) -> Result> { trace!("DurableWal::find_frame(page_no: {:?})", page_no); - if let Some(frame_no) = self.page_frames.get(&page_no) { - return Ok(Some(*frame_no)); - } let rt = tokio::runtime::Handle::current(); let req = rpc::FindFrameReq { page_no: page_no.get() as u64, @@ -131,14 +128,16 @@ impl Wal for DurableWal { .frame_no .map(|page_no| std::num::NonZeroU32::new(page_no as u32)) .flatten(); - if let Some(frame_no) = frame_no { - self.page_frames.insert(page_no, frame_no); - } Ok(frame_no) } fn read_frame(&mut self, frame_no: std::num::NonZeroU32, buffer: &mut [u8]) -> Result<()> { trace!("DurableWal::read_frame(frame_no: {:?})", frame_no); + // check if the frame exists in the local cache + if let Some(frame) = self.page_frames.get(&frame_no) { + buffer.copy_from_slice(&frame); + return Ok(()); + } let rt = tokio::runtime::Handle::current(); let frame_no = frame_no.get() as u64; let req = rpc::ReadFrameReq { frame_no }; @@ -146,6 +145,8 @@ impl Wal for DurableWal { let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); let frame = resp.into_inner().frame.unwrap(); buffer.copy_from_slice(&frame); + self.page_frames + .insert(std::num::NonZeroU32::new(frame_no as u32).unwrap(), frame); Ok(()) } @@ -176,7 +177,8 @@ impl Wal for DurableWal { } fn undo(&mut self, handler: Option<&mut U>) -> Result<()> { - todo!() + // TODO: no op + Ok(()) } fn savepoint(&mut self, rollback_data: &mut [u32]) { From d2ffdf71b07fc6f564ace169c49a5761e9a0eebe Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Wed, 3 Apr 2024 17:13:59 +0530 Subject: [PATCH 07/60] increase txn timeout --- libsql-server/src/connection/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsql-server/src/connection/mod.rs b/libsql-server/src/connection/mod.rs index fc0835c426..7fb07c00f7 100644 --- a/libsql-server/src/connection/mod.rs +++ b/libsql-server/src/connection/mod.rs @@ -25,7 +25,7 @@ pub mod program; pub mod write_proxy; #[cfg(not(test))] -const TXN_TIMEOUT: Duration = Duration::from_secs(5); +const TXN_TIMEOUT: Duration = Duration::from_secs(500); #[cfg(test)] const TXN_TIMEOUT: Duration = Duration::from_millis(100); From 796131afd464417eb07bf1d32ff748aacd898f01 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Wed, 3 Apr 2024 17:16:40 +0530 Subject: [PATCH 08/60] implement a shitty mutex lock manager --- Cargo.lock | 2 + libsql-server/src/lib.rs | 4 +- libsql-server/src/namespace/mod.rs | 8 +- .../src/namespace/replication_wal.rs | 6 +- libsql-storage/Cargo.toml | 4 +- libsql-storage/src/lib.rs | 80 ++++++++++++++++--- 6 files changed, 90 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9f957c112a..08217f8631 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2893,6 +2893,7 @@ name = "libsql-storage" version = "0.2.0" dependencies = [ "libsql-sys", + "log", "prost", "prost-build", "sieve-cache", @@ -2900,6 +2901,7 @@ dependencies = [ "tonic", "tonic-build", "tracing", + "uuid", ] [[package]] diff --git a/libsql-server/src/lib.rs b/libsql-server/src/lib.rs index 5073feb7b1..516d6a63da 100644 --- a/libsql-server/src/lib.rs +++ b/libsql-server/src/lib.rs @@ -4,7 +4,7 @@ use std::future::Future; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::str::FromStr; -use std::sync::{Arc, Weak}; +use std::sync::{Arc, Mutex, Weak}; use crate::connection::{Connection, MakeConnection}; use crate::error::Error; @@ -54,6 +54,7 @@ pub mod rpc; pub mod version; pub use hrana::proto as hrana_proto; +use libsql_storage::LockManager; mod database; mod error; @@ -547,6 +548,7 @@ where max_concurrent_connections: Arc::new(Semaphore::new(self.max_concurrent_connections)), scripted_backup, max_concurrent_requests: self.db_config.max_concurrent_requests, + lock_manager: Arc::new(Mutex::new(LockManager::new())), }; let factory = PrimaryNamespaceMaker::new(conf); diff --git a/libsql-server/src/namespace/mod.rs b/libsql-server/src/namespace/mod.rs index 8a0fb24d73..fae5d71364 100644 --- a/libsql-server/src/namespace/mod.rs +++ b/libsql-server/src/namespace/mod.rs @@ -52,6 +52,7 @@ use crate::{ use crate::namespace::fork::PointInTimeRestore; pub use fork::ForkError; +use libsql_storage::LockManager; use self::fork::ForkTask; use self::meta_store::{MetaStore, MetaStoreHandle}; @@ -1018,6 +1019,7 @@ pub struct PrimaryNamespaceConfig { pub(crate) max_concurrent_connections: Arc, pub(crate) scripted_backup: Option, pub(crate) max_concurrent_requests: u64, + pub(crate) lock_manager: Arc>, } pub type DumpStream = @@ -1149,7 +1151,11 @@ impl Namespace { ) .await?; - let wal_manager = make_replication_wal(bottomless_replicator, logger.clone()); + let wal_manager = make_replication_wal( + bottomless_replicator, + logger.clone(), + config.lock_manager.clone(), + ); let connection_maker: Arc<_> = MakeLibSqlConn::new( db_path.clone(), wal_manager.clone(), diff --git a/libsql-server/src/namespace/replication_wal.rs b/libsql-server/src/namespace/replication_wal.rs index 2e4d4d1be8..ae7867d24d 100644 --- a/libsql-server/src/namespace/replication_wal.rs +++ b/libsql-server/src/namespace/replication_wal.rs @@ -1,7 +1,8 @@ -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use bottomless::bottomless_wal::BottomlessWalWrapper; use bottomless::replicator::Replicator; +use libsql_storage::LockManager; use libsql_sys::wal::wrapper::{WalWrapper, WrappedWal}; use crate::replication::primary::replication_logger_wal::{ @@ -16,8 +17,9 @@ pub type ReplicationWal = WrappedWal, ReplicationLo pub fn make_replication_wal( bottomless: Option, logger: Arc, + lock_manager: Arc>, ) -> ReplicationWalManager { - let wal_manager = libsql_storage::DurableWalManager::new(); + let wal_manager = libsql_storage::DurableWalManager::new(lock_manager); WalWrapper::new( bottomless.map(|b| BottomlessWalWrapper::new(Arc::new(std::sync::Mutex::new(Some(b))))), ReplicationLoggerWalManager::new(wal_manager, logger), diff --git a/libsql-storage/Cargo.toml b/libsql-storage/Cargo.toml index 14a4f01f52..0d4abf4458 100644 --- a/libsql-storage/Cargo.toml +++ b/libsql-storage/Cargo.toml @@ -13,7 +13,9 @@ sieve-cache = "0.1.4" tokio = { version = "1.22.2", features = ["rt-multi-thread", "net", "io-std", "io-util", "time", "macros", "sync", "fs"] } tonic = { version = "0.10.0", features = ["tls"] } tracing = { version = "0.1.37", default-features = false } - +uuid = "1.7.0" +log = "0.4.20" + [dev-dependencies] prost-build = "0.12.0" tonic-build = "0.10" diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index eef2b9eca5..51ae5eac2e 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -1,5 +1,8 @@ +use libsql_sys::ffi::SQLITE_BUSY; +use libsql_sys::rusqlite; use libsql_sys::wal::{Result, Vfs, Wal, WalManager}; use sieve_cache::SieveCache; +use std::sync::{Arc, Mutex}; use tonic::transport::Channel; use tracing::trace; @@ -11,11 +14,13 @@ pub mod rpc { use rpc::storage_client::StorageClient; #[derive(Clone)] -pub struct DurableWalManager {} +pub struct DurableWalManager { + lock_manager: Arc>, +} impl DurableWalManager { - pub fn new() -> Self { - Self {} + pub fn new(lock_manager: Arc>) -> Self { + Self { lock_manager } } } @@ -37,7 +42,7 @@ impl WalManager for DurableWalManager { ) -> Result { let db_path = db_path.to_str().unwrap(); trace!("DurableWalManager::open(db_path: {})", db_path); - Ok(DurableWal::new()) + Ok(DurableWal::new(self.lock_manager.clone())) } fn close( @@ -75,10 +80,12 @@ pub struct DurableWal { client: StorageClient, page_frames: SieveCache>, db_size: u32, + name: String, + lock_manager: Arc>, } impl DurableWal { - fn new() -> Self { + fn new(lock_manager: Arc>) -> Self { let rt = tokio::runtime::Handle::current(); let client = StorageClient::connect("http://127.0.0.1:5002"); let mut client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); @@ -94,6 +101,8 @@ impl DurableWal { client, page_frames, db_size, + name: uuid::Uuid::new_v4().to_string(), + lock_manager, } } } @@ -167,12 +176,29 @@ impl Wal for DurableWal { } fn begin_write_txn(&mut self) -> Result<()> { - trace!("DurableWal::begin_write_txn()"); + let mut lock_manager = self.lock_manager.lock().unwrap(); + if !lock_manager.lock("default".to_string(), self.name.clone()) { + trace!( + "DurableWal::begin_write_txn() lock = false, id = {}", + self.name + ); + return Err(rusqlite::ffi::Error::new(SQLITE_BUSY)); + }; + trace!( + "DurableWal::begin_write_txn() lock = true, id = {}", + self.name + ); Ok(()) } fn end_write_txn(&mut self) -> Result<()> { - trace!("DurableWal::end_write_txn()"); + // release lock + let mut lock_manager = self.lock_manager.lock().unwrap(); + trace!( + "DurableWal::end_write_txn() id = {}, success = {}", + self.name, + lock_manager.unlock("default".to_string(), self.name.clone()) + ); Ok(()) } @@ -197,6 +223,7 @@ impl Wal for DurableWal { is_commit: bool, sync_flags: std::ffi::c_int, ) -> Result { + trace!("name = {}", self.name); trace!("DurableWal::insert_frames(page_size: {}, size_after: {}, is_commit: {}, sync_flags: {})", page_size, size_after, is_commit, sync_flags); let rt = tokio::runtime::Handle::current(); let frames = page_headers @@ -252,12 +279,13 @@ impl Wal for DurableWal { } fn frames_in_wal(&mut self) -> u32 { - trace!("DurableWal::frames_in_wal()"); let rt = tokio::runtime::Handle::current(); let req = rpc::FramesInWalReq {}; let resp = self.client.frames_in_wal(req); let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); - resp.into_inner().count + let count = resp.into_inner().count; + trace!("DurableWal::frames_in_wal() = {}", count); + count } fn backfilled(&self) -> u32 { @@ -268,3 +296,37 @@ impl Wal for DurableWal { todo!() } } + +pub struct LockManager { + locks: std::collections::HashMap, +} + +impl LockManager { + pub fn new() -> Self { + Self { + locks: std::collections::HashMap::new(), + } + } + + pub fn lock(&mut self, namespace: String, wal_id: String) -> bool { + if let Some(lock) = self.locks.get(&namespace) { + if lock == &wal_id { + return true; + } + return false; + } + self.locks.insert(namespace, wal_id); + true + } + + pub fn unlock(&mut self, namespace: String, wal_id: String) -> bool { + if let Some(lock) = self.locks.get(&namespace) { + if lock == &wal_id { + self.locks.remove(&namespace); + return true; + } + return false; + } + true + } +} From 66273847345f6db20edd468f045d2d9abbe09990 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Wed, 3 Apr 2024 21:46:56 +0530 Subject: [PATCH 09/60] Reset WAL interface changes and use mutex for RPC client --- Cargo.lock | 1 + .../src/injector/injector_wal.rs | 4 +-- .../primary/replication_logger_wal.rs | 4 +-- libsql-storage/Cargo.toml | 5 ++-- libsql-storage/src/lib.rs | 28 ++++++++++++------- libsql-sys/src/wal/either.rs | 4 +-- libsql-sys/src/wal/mod.rs | 4 +-- libsql-sys/src/wal/sqlite3_wal.rs | 4 +-- libsql-sys/src/wal/wrapper.rs | 14 +++++----- 9 files changed, 39 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 08217f8631..3fe4b41287 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2894,6 +2894,7 @@ version = "0.2.0" dependencies = [ "libsql-sys", "log", + "parking_lot", "prost", "prost-build", "sieve-cache", diff --git a/libsql-replication/src/injector/injector_wal.rs b/libsql-replication/src/injector/injector_wal.rs index 3b4f82ddeb..3f2da8aab9 100644 --- a/libsql-replication/src/injector/injector_wal.rs +++ b/libsql-replication/src/injector/injector_wal.rs @@ -218,7 +218,7 @@ impl Wal for InjectorWal { self.inner.callback() } - fn frames_in_wal(&mut self) -> u32 { + fn frames_in_wal(&self) -> u32 { self.inner.frames_in_wal() } @@ -230,7 +230,7 @@ impl Wal for InjectorWal { self.inner.backfilled() } - fn frame_page_no(&mut self, frame_no: NonZeroU32) -> Option { + fn frame_page_no(&self, frame_no: NonZeroU32) -> Option { self.inner.frame_page_no(frame_no) } } diff --git a/libsql-server/src/replication/primary/replication_logger_wal.rs b/libsql-server/src/replication/primary/replication_logger_wal.rs index 4d6167cd86..7b7bb0074e 100644 --- a/libsql-server/src/replication/primary/replication_logger_wal.rs +++ b/libsql-server/src/replication/primary/replication_logger_wal.rs @@ -268,7 +268,7 @@ impl Wal for ReplicationLoggerWal { self.inner.callback() } - fn frames_in_wal(&mut self) -> u32 { + fn frames_in_wal(&self) -> u32 { self.inner.frames_in_wal() } @@ -280,7 +280,7 @@ impl Wal for ReplicationLoggerWal { self.inner.backfilled() } - fn frame_page_no(&mut self, frame_no: NonZeroU32) -> Option { + fn frame_page_no(&self, frame_no: NonZeroU32) -> Option { self.inner.frame_page_no(frame_no) } } diff --git a/libsql-storage/Cargo.toml b/libsql-storage/Cargo.toml index 0d4abf4458..395ec853a2 100644 --- a/libsql-storage/Cargo.toml +++ b/libsql-storage/Cargo.toml @@ -7,14 +7,15 @@ repository = "https://github.com/tursodatabase/libsql" license = "MIT" [dependencies] -libsql-sys = { path = "../libsql-sys" } +libsql-sys = { path = "../libsql-sys", features = ["rusqlite"] } prost = "0.12" sieve-cache = "0.1.4" tokio = { version = "1.22.2", features = ["rt-multi-thread", "net", "io-std", "io-util", "time", "macros", "sync", "fs"] } tonic = { version = "0.10.0", features = ["tls"] } tracing = { version = "0.1.37", default-features = false } -uuid = "1.7.0" +uuid = { version = "1.7.0", features = ["v4"] } log = "0.4.20" +parking_lot = "0.12.1" [dev-dependencies] prost-build = "0.12.0" diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 51ae5eac2e..f938b83047 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -1,3 +1,5 @@ +// use libsql_sys::ffi::SQLITE_BUSY; +// use libsql_sys::rusqlite; use libsql_sys::ffi::SQLITE_BUSY; use libsql_sys::rusqlite; use libsql_sys::wal::{Result, Vfs, Wal, WalManager}; @@ -5,6 +7,7 @@ use sieve_cache::SieveCache; use std::sync::{Arc, Mutex}; use tonic::transport::Channel; use tracing::trace; +use uuid::uuid; pub mod rpc { #![allow(clippy::all)] @@ -77,7 +80,7 @@ impl WalManager for DurableWalManager { } pub struct DurableWal { - client: StorageClient, + client: parking_lot::Mutex>, page_frames: SieveCache>, db_size: u32, name: String, @@ -98,7 +101,7 @@ impl DurableWal { let page_frames = SieveCache::new(1000).unwrap(); Self { - client, + client: parking_lot::Mutex::new(client), page_frames, db_size, name: uuid::Uuid::new_v4().to_string(), @@ -130,7 +133,8 @@ impl Wal for DurableWal { let req = rpc::FindFrameReq { page_no: page_no.get() as u64, }; - let resp = self.client.find_frame(req); + let mut binding = self.client.lock(); + let resp = binding.find_frame(req); let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); let frame_no = resp .into_inner() @@ -150,7 +154,8 @@ impl Wal for DurableWal { let rt = tokio::runtime::Handle::current(); let frame_no = frame_no.get() as u64; let req = rpc::ReadFrameReq { frame_no }; - let resp = self.client.read_frame(req); + let mut binding = self.client.lock(); + let resp = binding.read_frame(req); let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); let frame = resp.into_inner().frame.unwrap(); buffer.copy_from_slice(&frame); @@ -159,12 +164,13 @@ impl Wal for DurableWal { Ok(()) } - fn frame_page_no(&mut self, frame_no: std::num::NonZeroU32) -> Option { + fn frame_page_no(&self, frame_no: std::num::NonZeroU32) -> Option { trace!("DurableWal::frame_page_no(frame_no: {:?})", frame_no); let rt = tokio::runtime::Handle::current(); let frame_no = frame_no.get() as u64; let req = rpc::FramePageNumReq { frame_no }; - let resp = self.client.frame_page_num(req); + let mut binding = self.client.lock(); + let resp = binding.frame_page_num(req); let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); let page_no = resp.into_inner().page_no; std::num::NonZeroU32::new(page_no as u32) @@ -195,7 +201,7 @@ impl Wal for DurableWal { // release lock let mut lock_manager = self.lock_manager.lock().unwrap(); trace!( - "DurableWal::end_write_txn() id = {}, success = {}", + "DurableWal::end_write_txn() id = {}, unlocked = {}", self.name, lock_manager.unlock("default".to_string(), self.name.clone()) ); @@ -237,7 +243,8 @@ impl Wal for DurableWal { }) .collect(); let req = rpc::InsertFramesReq { frames }; - let resp = self.client.insert_frames(req); + let mut binding = self.client.lock(); + let resp = binding.insert_frames(req); let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); self.db_size = size_after; Ok(resp.into_inner().num_frames as usize) @@ -278,10 +285,11 @@ impl Wal for DurableWal { 0 } - fn frames_in_wal(&mut self) -> u32 { + fn frames_in_wal(&self) -> u32 { let rt = tokio::runtime::Handle::current(); let req = rpc::FramesInWalReq {}; - let resp = self.client.frames_in_wal(req); + let mut binding = self.client.lock(); + let resp = binding.frames_in_wal(req); let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); let count = resp.into_inner().count; trace!("DurableWal::frames_in_wal() = {}", count); diff --git a/libsql-sys/src/wal/either.rs b/libsql-sys/src/wal/either.rs index f0f8bda43f..683a9a8cd6 100644 --- a/libsql-sys/src/wal/either.rs +++ b/libsql-sys/src/wal/either.rs @@ -176,7 +176,7 @@ where } } - fn frames_in_wal(&mut self) -> u32 { + fn frames_in_wal(&self) -> u32 { match self { Either::Left(l) => l.frames_in_wal(), Either::Right(r) => r.frames_in_wal(), @@ -197,7 +197,7 @@ where } } - fn frame_page_no(&mut self, frame_no: std::num::NonZeroU32) -> Option { + fn frame_page_no(&self, frame_no: std::num::NonZeroU32) -> Option { match self { Either::Left(l) => l.frame_page_no(frame_no), Either::Right(r) => r.frame_page_no(frame_no), diff --git a/libsql-sys/src/wal/mod.rs b/libsql-sys/src/wal/mod.rs index 2a682bde43..db8d11b38c 100644 --- a/libsql-sys/src/wal/mod.rs +++ b/libsql-sys/src/wal/mod.rs @@ -190,7 +190,7 @@ pub trait Wal { fn find_frame(&mut self, page_no: NonZeroU32) -> Result>; /// reads frame `frame_no` into buffer. fn read_frame(&mut self, frame_no: NonZeroU32, buffer: &mut [u8]) -> Result<()>; - fn frame_page_no(&mut self, frame_no: NonZeroU32) -> Option; + fn frame_page_no(&self, frame_no: NonZeroU32) -> Option; fn db_size(&self) -> u32; @@ -237,7 +237,7 @@ pub trait Wal { /// the last call, then return 0. fn callback(&self) -> i32; - fn frames_in_wal(&mut self) -> u32; + fn frames_in_wal(&self) -> u32; fn backfilled(&self) -> u32; fn db_file(&self) -> &Sqlite3File; } diff --git a/libsql-sys/src/wal/sqlite3_wal.rs b/libsql-sys/src/wal/sqlite3_wal.rs index 651f0c2eca..6d68b81ad3 100644 --- a/libsql-sys/src/wal/sqlite3_wal.rs +++ b/libsql-sys/src/wal/sqlite3_wal.rs @@ -409,7 +409,7 @@ impl Wal for Sqlite3Wal { unsafe { (self.inner.methods.xCallback.unwrap())(self.inner.pData) } } - fn frames_in_wal(&mut self) -> u32 { + fn frames_in_wal(&self) -> u32 { unsafe { let wal = &*(self.inner.pData as *const sqlite3_wal); wal.hdr.mxFrame @@ -435,7 +435,7 @@ impl Wal for Sqlite3Wal { /// ported from wal.c /// returns the page_no corresponding to a given frame_no - fn frame_page_no(&mut self, frame_no: NonZeroU32) -> Option { + fn frame_page_no(&self, frame_no: NonZeroU32) -> Option { unsafe { let ptr = &mut (*(self.inner.pData as *mut sqlite3_wal)); NonZeroU32::new(libsql_ffi::sqlite3_wal_frame_page_no(ptr, frame_no.get())) diff --git a/libsql-sys/src/wal/wrapper.rs b/libsql-sys/src/wal/wrapper.rs index 758dc85b2d..08af672f00 100644 --- a/libsql-sys/src/wal/wrapper.rs +++ b/libsql-sys/src/wal/wrapper.rs @@ -199,8 +199,8 @@ where self.wrapper.callback(&self.wrapped) } - fn frames_in_wal(&mut self) -> u32 { - self.wrapper.frames_in_wal(&mut self.wrapped) + fn frames_in_wal(&self) -> u32 { + self.wrapper.frames_in_wal(&self.wrapped) } fn db_file(&self) -> &super::Sqlite3File { @@ -211,8 +211,8 @@ where self.wrapped.backfilled() } - fn frame_page_no(&mut self, frame_no: NonZeroU32) -> Option { - self.wrapper.frame_page_no(&mut self.wrapped, frame_no) + fn frame_page_no(&self, frame_no: NonZeroU32) -> Option { + self.wrapper.frame_page_no(&self.wrapped, frame_no) } } @@ -329,7 +329,7 @@ pub trait WrapWal { wrapped.callback() } - fn frames_in_wal(&self, wrapped: &mut W) -> u32 { + fn frames_in_wal(&self, wrapped: &W) -> u32 { wrapped.frames_in_wal() } @@ -356,7 +356,7 @@ pub trait WrapWal { manager.close(wrapped, db, sync_flags, scratch) } - fn frame_page_no(&mut self, wrapped: &mut W, frame_no: NonZeroU32) -> Option { + fn frame_page_no(&self, wrapped: &W, frame_no: NonZeroU32) -> Option { wrapped.frame_page_no(frame_no) } } @@ -542,7 +542,7 @@ impl, W: Wal> WrapWal for Option { } } - fn frames_in_wal(&self, wrapped: &mut W) -> u32 { + fn frames_in_wal(&self, wrapped: &W) -> u32 { match self { Some(t) => t.frames_in_wal(wrapped), None => wrapped.frames_in_wal(), From f588889558851879be87e6537aad636da8ceadd8 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 4 Apr 2024 00:05:11 +0530 Subject: [PATCH 10/60] Add working docker --- Dockerfile.ss | 17 +++++++++++++++++ libsql-storage-server/src/main.rs | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.ss diff --git a/Dockerfile.ss b/Dockerfile.ss new file mode 100644 index 0000000000..d24bcacde7 --- /dev/null +++ b/Dockerfile.ss @@ -0,0 +1,17 @@ +FROM rust:latest as builder + +WORKDIR /usr/src/app +COPY . . +# Will build and cache the binary and dependent crates in release mode +RUN --mount=type=cache,target=/usr/local/cargo,from=rust:latest,source=/usr/local/cargo \ + --mount=type=cache,target=target \ + cargo build --bin libsql-storage-server && mv ./target/debug/libsql-storage-server ./libsql-storage-server && chmod +x ./libsql-storage-server + + +# Runtime image +FROM debian:bookworm-slim +EXPOSE 5002 + +COPY --from=builder /usr/src/app/libsql-storage-server / + +CMD ["/libsql-storage-server"] \ No newline at end of file diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index fb1409a1c3..a102a7065e 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -25,7 +25,7 @@ struct Cli { #[clap( long, env = "LIBSQL_STORAGE_LISTEN_ADDR", - default_value = "127.0.0.1:5002" + default_value = "0.0.0.0:5002" )] listen_addr: SocketAddr, } From 03c4ddb4b735346741a6291eaf7e3981885480e2 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 4 Apr 2024 00:05:11 +0530 Subject: [PATCH 11/60] Add fly config and move the Dockerfile --- .../Dockerfile | 11 ++++++---- libsql-storage-server/fly.toml | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) rename Dockerfile.ss => libsql-storage-server/Dockerfile (69%) create mode 100644 libsql-storage-server/fly.toml diff --git a/Dockerfile.ss b/libsql-storage-server/Dockerfile similarity index 69% rename from Dockerfile.ss rename to libsql-storage-server/Dockerfile index d24bcacde7..654aca40c1 100644 --- a/Dockerfile.ss +++ b/libsql-storage-server/Dockerfile @@ -1,17 +1,20 @@ +# run this from the root +# docker build -f libsql-storage-server/Dockerfile -t lss . +# docker run -p lss + FROM rust:latest as builder -WORKDIR /usr/src/app +WORKDIR /app COPY . . -# Will build and cache the binary and dependent crates in release mode + RUN --mount=type=cache,target=/usr/local/cargo,from=rust:latest,source=/usr/local/cargo \ --mount=type=cache,target=target \ cargo build --bin libsql-storage-server && mv ./target/debug/libsql-storage-server ./libsql-storage-server && chmod +x ./libsql-storage-server -# Runtime image FROM debian:bookworm-slim EXPOSE 5002 -COPY --from=builder /usr/src/app/libsql-storage-server / +COPY --from=builder /app/libsql-storage-server / CMD ["/libsql-storage-server"] \ No newline at end of file diff --git a/libsql-storage-server/fly.toml b/libsql-storage-server/fly.toml new file mode 100644 index 0000000000..45beffae72 --- /dev/null +++ b/libsql-storage-server/fly.toml @@ -0,0 +1,20 @@ +# fly deploy -c libsql-storage-server/fly.toml --dockerfile libsql-storage-server/Dockerfile + +app = 'libsql-storage-server' +primary_region = 'ams' + +[build] + +[[services]] +http_checks = [] +internal_port = 5002 +processes = ["app"] +protocol = "tcp" +script_checks = [] +auto_stop_machines = true +auto_start_machines = true +[[services.ports]] +port = 5002 + +[[vm]] +size = 'shared-cpu-1x' From 4c88315c6b813ecfc319270c3907c9cc6bdd1e97 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 4 Apr 2024 00:15:11 +0530 Subject: [PATCH 12/60] simplify fly config --- libsql-storage-server/fly.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/libsql-storage-server/fly.toml b/libsql-storage-server/fly.toml index 45beffae72..0790cd4a81 100644 --- a/libsql-storage-server/fly.toml +++ b/libsql-storage-server/fly.toml @@ -11,8 +11,6 @@ internal_port = 5002 processes = ["app"] protocol = "tcp" script_checks = [] -auto_stop_machines = true -auto_start_machines = true [[services.ports]] port = 5002 From b10dd9b97bf305f904dbea87644923dbd08a5a4e Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 4 Apr 2024 00:45:14 +0530 Subject: [PATCH 13/60] make the storage server configurable and use the fly version --- libsql-storage-server/src/main.rs | 6 +----- libsql-storage/src/lib.rs | 7 ++++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index a102a7065e..466d86d012 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -22,11 +22,7 @@ use tracing_subscriber::{EnvFilter, FmtSubscriber}; #[command(about = "libSQL storage server", version = Version::default(), long_about = None)] struct Cli { /// The address and port the storage RPC protocol listens to. Example: `127.0.0.1:5002`. - #[clap( - long, - env = "LIBSQL_STORAGE_LISTEN_ADDR", - default_value = "0.0.0.0:5002" - )] + #[clap(long, env = "LIBSQL_STORAGE_LISTEN_ADDR", default_value = "[::]:5002")] listen_addr: SocketAddr, } diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index f938b83047..df261e305d 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -90,7 +90,12 @@ pub struct DurableWal { impl DurableWal { fn new(lock_manager: Arc>) -> Self { let rt = tokio::runtime::Handle::current(); - let client = StorageClient::connect("http://127.0.0.1:5002"); + // connect to ss on fly + // export LIBSQL_STORAGE_SERVER_ADDR=http://libsql-storage-server.internal:5002 + let address = std::env::var("LIBSQL_STORAGE_SERVER_ADDR") + .unwrap_or("http://127.0.0.1:5002".to_string()); + trace!("DurableWal::new() address = {}", address); + let client = StorageClient::connect(address); let mut client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); let req = rpc::DbSizeReq {}; From 179c300b99c067cd7881117bc503c81f96955766 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 4 Apr 2024 08:45:24 +0530 Subject: [PATCH 14/60] Add store trait --- libsql-storage-server/src/main.rs | 15 ++++++++++----- libsql-storage-server/src/store.rs | 7 +++++++ 2 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 libsql-storage-server/src/store.rs diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 466d86d012..de1706f41c 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -1,3 +1,6 @@ +mod store; + +use crate::store::Store; use anyhow::Result; use clap::Parser; use libsql_storage::rpc::storage_server::{Storage, StorageServer}; @@ -46,9 +49,11 @@ impl FrameStore { pub fn new() -> Self { Self::default() } +} +impl Store for FrameStore { // inserts a new frame for the page number and returns the new frame value - pub fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64 { + fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64 { let frame_no = self.max_frame_no + 1; self.max_frame_no = frame_no; self.frames.insert( @@ -65,23 +70,23 @@ impl FrameStore { frame_no } - pub fn read_frame(&self, frame_no: u64) -> Option<&bytes::Bytes> { + fn read_frame(&self, frame_no: u64) -> Option<&bytes::Bytes> { self.frames.get(&frame_no).map(|frame| &frame.data) } // given a page number, return the maximum frame for the page - pub fn find_frame(&self, page_no: u64) -> Option { + fn find_frame(&self, page_no: u64) -> Option { self.pages .get(&page_no) .map(|frames| *frames.last().unwrap()) } // given a frame num, return the page number - pub fn frame_page_no(&self, frame_no: u64) -> Option { + fn frame_page_no(&self, frame_no: u64) -> Option { self.frames.get(&frame_no).map(|frame| frame.page_no) } - pub fn frames_in_wal(&self) -> u64 { + fn frames_in_wal(&self) -> u64 { self.max_frame_no } } diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs new file mode 100644 index 0000000000..3dcee8ed46 --- /dev/null +++ b/libsql-storage-server/src/store.rs @@ -0,0 +1,7 @@ +pub trait Store { + fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64; + fn read_frame(&self, frame_no: u64) -> Option<&bytes::Bytes>; + fn find_frame(&self, page_no: u64) -> Option; + fn frame_page_no(&self, frame_no: u64) -> Option; + fn frames_in_wal(&self) -> u64; +} From 3fe9f6a104662077fecfa9798fd7c3fec89bf7bc Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 4 Apr 2024 08:45:24 +0530 Subject: [PATCH 15/60] rename traits and struct --- libsql-storage-server/src/main.rs | 10 +++++----- libsql-storage-server/src/store.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index de1706f41c..dd82c865ce 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -1,6 +1,6 @@ mod store; -use crate::store::Store; +use crate::store::FrameStore; use anyhow::Result; use clap::Parser; use libsql_storage::rpc::storage_server::{Storage, StorageServer}; @@ -37,7 +37,7 @@ struct FrameData { } #[derive(Default)] -struct FrameStore { +struct InMemFrameStore { // contains a frame data, key is the frame number frames: BTreeMap, // pages map contains the page number as a key and the list of frames for the page as a value @@ -45,13 +45,13 @@ struct FrameStore { max_frame_no: u64, } -impl FrameStore { +impl InMemFrameStore { pub fn new() -> Self { Self::default() } } -impl Store for FrameStore { +impl FrameStore for InMemFrameStore { // inserts a new frame for the page number and returns the new frame value fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64 { let frame_no = self.max_frame_no + 1; @@ -93,7 +93,7 @@ impl Store for FrameStore { #[derive(Default)] struct Service { - store: Arc>, + store: Arc>, db_size: AtomicU32, } diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index 3dcee8ed46..e475735617 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -1,4 +1,4 @@ -pub trait Store { +pub trait FrameStore { fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64; fn read_frame(&self, frame_no: u64) -> Option<&bytes::Bytes>; fn find_frame(&self, page_no: u64) -> Option; From 300d5f7a75a9a2808496603fd155afd4ceb209bd Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 4 Apr 2024 12:54:42 +0530 Subject: [PATCH 16/60] Add redis framestore impl --- Cargo.lock | 32 ++++++++++ libsql-storage-server/Cargo.toml | 1 + libsql-storage-server/src/main.rs | 99 +++++++++++++++++++++++++++--- libsql-storage-server/src/store.rs | 2 +- 4 files changed, 125 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3fe4b41287..bdd02a313d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1293,6 +1293,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "console" version = "0.15.8" @@ -2913,6 +2923,7 @@ dependencies = [ "bytes", "clap 4.4.18", "libsql-storage", + "redis", "tokio", "tonic", "tracing", @@ -3988,6 +3999,21 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redis" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6472825949c09872e8f2c50bde59fcefc17748b6be5c90fd67cd8b4daca73bfd" +dependencies = [ + "combine", + "itoa", + "percent-encoding", + "ryu", + "sha1_smol", + "socket2", + "url", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -4529,6 +4555,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.10.8" diff --git a/libsql-storage-server/Cargo.toml b/libsql-storage-server/Cargo.toml index e480c414f3..9c92eba1eb 100644 --- a/libsql-storage-server/Cargo.toml +++ b/libsql-storage-server/Cargo.toml @@ -13,6 +13,7 @@ anyhow = "1.0.66" bytes = "1.5.0" clap = { version = "4.0.23", features = [ "derive", "env", "string" ] } libsql-storage = { path = "../libsql-storage" } +redis = "0.25.3" tokio = { version = "1.22.2", features = ["rt-multi-thread", "net", "io-std", "io-util", "time", "macros", "sync", "fs", "signal"] } tonic = { version = "0.10.0", features = ["tls"] } tracing = "0.1.37" diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index dd82c865ce..5af37dc71d 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -2,6 +2,7 @@ mod store; use crate::store::FrameStore; use anyhow::Result; +use bytes::Bytes; use clap::Parser; use libsql_storage::rpc::storage_server::{Storage, StorageServer}; use libsql_storage::rpc::{ @@ -10,7 +11,9 @@ use libsql_storage::rpc::{ ReadFrameResp, }; use libsql_storage_server::version::Version; +use redis::{Client, Commands}; use std::collections::BTreeMap; +use std::fmt::format; use std::iter::Map; use std::net::SocketAddr; use std::sync::atomic::AtomicU32; @@ -70,8 +73,8 @@ impl FrameStore for InMemFrameStore { frame_no } - fn read_frame(&self, frame_no: u64) -> Option<&bytes::Bytes> { - self.frames.get(&frame_no).map(|frame| &frame.data) + fn read_frame(&self, frame_no: u64) -> Option { + self.frames.get(&frame_no).map(|frame| frame.data.clone()) } // given a page number, return the maximum frame for the page @@ -91,15 +94,95 @@ impl FrameStore for InMemFrameStore { } } -#[derive(Default)] +struct RedisFrameStore { + client: Client, +} + +impl RedisFrameStore { + pub fn new(client: Client) -> Self { + Self { client } + } +} + +impl FrameStore for RedisFrameStore { + fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { + let namespace = "default"; + let max_frame_key = format!("{}/max_frame_no", namespace); + let mut con = self.client.get_connection().unwrap(); + let max_frame_no: i64 = redis::cmd("INCR") + .arg(max_frame_key.clone()) + .query(&mut con) + .unwrap(); + let frame_key = format!("f/{}/{}", namespace, max_frame_no); + redis::cmd("HSET") + .arg(frame_key) + .arg("f") + .arg(frame.to_vec()) + .arg("p") + .arg(page_no) + .execute(&mut con); + let page_key = format!("p/{}/{}", namespace, page_no); + redis::cmd("SET") + .arg(page_key) + .arg(max_frame_no) + .execute(&mut con); + max_frame_no as u64 + } + + fn read_frame(&self, frame_no: u64) -> Option { + let namespace = "default"; + let frame_key = format!("f/{}/{}", namespace, frame_no); + let mut con = self.client.get_connection().unwrap(); + let frame: Vec = redis::cmd("HGET") + .arg(frame_key) + .arg("f") + .query(&mut con) + .unwrap(); + Some(Bytes::from(frame)) + } + + fn find_frame(&self, page_no: u64) -> Option { + let page_key = format!("p/{}/{}", "default", page_no); + let mut con = self.client.get_connection().unwrap(); + let frame_no = redis::cmd("GET").arg(page_key).query(&mut con); + match frame_no { + Ok(frame_no) => Some(frame_no), + Err(_) => None, + } + } + + fn frame_page_no(&self, frame_no: u64) -> Option { + let namespace = "default"; + let frame_key = format!("f/{}/{}", namespace, frame_no); + let mut con = self.client.get_connection().unwrap(); + let page_no: i64 = redis::cmd("HGET") + .arg(frame_key) + .arg("p") + .query(&mut con) + .unwrap(); + Some(page_no as u64) + } + + fn frames_in_wal(&self) -> u64 { + let namespace = "default"; + let max_frame_key = format!("{}/max_frame_no", namespace); + let mut con = self.client.get_connection().unwrap(); + let max_frame_no = redis::cmd("GET").arg(max_frame_key.clone()).query(&mut con); + max_frame_no.unwrap_or_else(|_| 0) as u64 + } +} + struct Service { - store: Arc>, + store: Arc>, db_size: AtomicU32, } impl Service { - pub fn new() -> Self { - Self::default() + pub fn new(client: Client) -> Self { + Self { + store: Arc::new(Mutex::new(RedisFrameStore::new(client))), + db_size: AtomicU32::new(0), + } } } @@ -198,8 +281,8 @@ async fn main() -> Result<()> { .expect("setting default subscriber failed"); let args = Cli::parse(); - - let service = Service::default(); + let client = Client::open("redis://127.0.0.1/").unwrap(); + let service = Service::new(client); println!("Starting libSQL storage server on {}", args.listen_addr); trace!( diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index e475735617..44c02a4aac 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -1,6 +1,6 @@ pub trait FrameStore { fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64; - fn read_frame(&self, frame_no: u64) -> Option<&bytes::Bytes>; + fn read_frame(&self, frame_no: u64) -> Option; fn find_frame(&self, page_no: u64) -> Option; fn frame_page_no(&self, frame_no: u64) -> Option; fn frames_in_wal(&self) -> u64; From ab729a75674f0b01653975126420db2f12fcafc4 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 4 Apr 2024 13:10:28 +0530 Subject: [PATCH 17/60] handle nil responses from redis and return proper default --- libsql-storage-server/src/main.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 5af37dc71d..0e78b383e3 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -147,7 +147,12 @@ impl FrameStore for RedisFrameStore { let frame_no = redis::cmd("GET").arg(page_key).query(&mut con); match frame_no { Ok(frame_no) => Some(frame_no), - Err(_) => None, + Err(e) => { + if !is_nil_response(&e) { + error!("find_frame() failed for page_no={} with err={}", page_no, e); + } + None + } } } @@ -167,8 +172,13 @@ impl FrameStore for RedisFrameStore { let namespace = "default"; let max_frame_key = format!("{}/max_frame_no", namespace); let mut con = self.client.get_connection().unwrap(); - let max_frame_no = redis::cmd("GET").arg(max_frame_key.clone()).query(&mut con); - max_frame_no.unwrap_or_else(|_| 0) as u64 + let result = redis::cmd("GET").arg(max_frame_key.clone()).query(&mut con); + result.unwrap_or_else(|e| { + if !is_nil_response(&e) { + error!("frames_in_wal() failed with err={}", e); + } + 0 + }) as u64 } } @@ -296,3 +306,7 @@ async fn main() -> Result<()> { Ok(()) } + +fn is_nil_response(e: &redis::RedisError) -> bool { + e.to_string().contains("response was nil") +} From 762d558ea5511fe6148cc704f20043e554a57324 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Tue, 9 Apr 2024 13:17:08 +0530 Subject: [PATCH 18/60] improve redis code: transactions, commands, env var --- libsql-storage-server/src/main.rs | 92 +++++++++++++++++++------------ libsql-storage/src/lib.rs | 2 +- 2 files changed, 59 insertions(+), 35 deletions(-) diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 0e78b383e3..4c1b1fd329 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -11,7 +11,7 @@ use libsql_storage::rpc::{ ReadFrameResp, }; use libsql_storage_server::version::Version; -use redis::{Client, Commands}; +use redis::{Client, Commands, RedisResult}; use std::collections::BTreeMap; use std::fmt::format; use std::iter::Map; @@ -108,43 +108,58 @@ impl FrameStore for RedisFrameStore { fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { let namespace = "default"; let max_frame_key = format!("{}/max_frame_no", namespace); + let mut con = self.client.get_connection().unwrap(); - let max_frame_no: i64 = redis::cmd("INCR") - .arg(max_frame_key.clone()) - .query(&mut con) + // max_frame_key might change if another client inserts a frame, so do + // all this in a transaction! + let (max_frame_no,): (u64,) = + redis::transaction(&mut con, &[&max_frame_key], |con, pipe| { + let result: RedisResult = con.get(max_frame_key.clone()); + if result.is_err() && !is_nil_response(result.as_ref().err().unwrap()) { + return Err(result.err().unwrap()); + } + let max_frame_no = result.unwrap_or(0) + 1; + let frame_key = format!("f/{}/{}", namespace, max_frame_no); + let page_key = format!("p/{}/{}", namespace, page_no); + + pipe.hset::>(frame_key.clone(), "f", frame.to_vec()) + .ignore() + .hset::(frame_key.clone(), "p", page_no) + .ignore() + .set::(page_key, max_frame_no) + .ignore() + .set::(max_frame_key.clone(), max_frame_no) + .ignore() + .get(max_frame_key.clone()) + .query(con) + }) .unwrap(); - let frame_key = format!("f/{}/{}", namespace, max_frame_no); - redis::cmd("HSET") - .arg(frame_key) - .arg("f") - .arg(frame.to_vec()) - .arg("p") - .arg(page_no) - .execute(&mut con); - let page_key = format!("p/{}/{}", namespace, page_no); - redis::cmd("SET") - .arg(page_key) - .arg(max_frame_no) - .execute(&mut con); - max_frame_no as u64 + max_frame_no } fn read_frame(&self, frame_no: u64) -> Option { let namespace = "default"; let frame_key = format!("f/{}/{}", namespace, frame_no); let mut con = self.client.get_connection().unwrap(); - let frame: Vec = redis::cmd("HGET") - .arg(frame_key) - .arg("f") - .query(&mut con) - .unwrap(); - Some(Bytes::from(frame)) + let result = con.hget::>(frame_key.clone(), "f"); + match result { + Ok(frame) => Some(Bytes::from(frame)), + Err(e) => { + if !is_nil_response(&e) { + error!( + "read_frame() failed for frame_no={} with err={}", + frame_no, e + ); + } + None + } + } } fn find_frame(&self, page_no: u64) -> Option { let page_key = format!("p/{}/{}", "default", page_no); let mut con = self.client.get_connection().unwrap(); - let frame_no = redis::cmd("GET").arg(page_key).query(&mut con); + let frame_no = con.get::(page_key.clone()); match frame_no { Ok(frame_no) => Some(frame_no), Err(e) => { @@ -160,25 +175,32 @@ impl FrameStore for RedisFrameStore { let namespace = "default"; let frame_key = format!("f/{}/{}", namespace, frame_no); let mut con = self.client.get_connection().unwrap(); - let page_no: i64 = redis::cmd("HGET") - .arg(frame_key) - .arg("p") - .query(&mut con) - .unwrap(); - Some(page_no as u64) + let result = con.hget::(frame_key.clone(), "p"); + match result { + Ok(page_no) => Some(page_no), + Err(e) => { + if !is_nil_response(&e) { + error!( + "frame_page_no() failed for frame_no={} with err={}", + frame_no, e + ); + } + None + } + } } fn frames_in_wal(&self) -> u64 { let namespace = "default"; let max_frame_key = format!("{}/max_frame_no", namespace); let mut con = self.client.get_connection().unwrap(); - let result = redis::cmd("GET").arg(max_frame_key.clone()).query(&mut con); + let result = con.get::(max_frame_key.clone()); result.unwrap_or_else(|e| { if !is_nil_response(&e) { error!("frames_in_wal() failed with err={}", e); } 0 - }) as u64 + }) } } @@ -291,7 +313,9 @@ async fn main() -> Result<()> { .expect("setting default subscriber failed"); let args = Cli::parse(); - let client = Client::open("redis://127.0.0.1/").unwrap(); + // export REDIS_ADDR=http://libsql-storage-server.internal:5002 + let redis_addr = std::env::var("REDIS_ADDR").unwrap_or("redis://127.0.0.1/".to_string()); + let client = Client::open(redis_addr).unwrap(); let service = Service::new(client); println!("Starting libSQL storage server on {}", args.listen_addr); diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index df261e305d..c4508252ff 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -90,7 +90,7 @@ pub struct DurableWal { impl DurableWal { fn new(lock_manager: Arc>) -> Self { let rt = tokio::runtime::Handle::current(); - // connect to ss on fly + // connect to external storage server // export LIBSQL_STORAGE_SERVER_ADDR=http://libsql-storage-server.internal:5002 let address = std::env::var("LIBSQL_STORAGE_SERVER_ADDR") .unwrap_or("http://127.0.0.1:5002".to_string()); From ca9d5a56c2b7c7a6793fd394d2965cba53685888 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 11 Apr 2024 18:49:09 +0530 Subject: [PATCH 19/60] empty From 53da4e9510587eeb3c813bb377dcf5dfe6511c9d Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 11 Apr 2024 18:50:54 +0530 Subject: [PATCH 20/60] Add updated workflow image --- .github/workflows/server-pr-images.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/server-pr-images.yml b/.github/workflows/server-pr-images.yml index 1da1df7186..6f5e245027 100644 --- a/.github/workflows/server-pr-images.yml +++ b/.github/workflows/server-pr-images.yml @@ -4,8 +4,9 @@ name: libsql server pull request Docker image on: + workflow_dispatch: pull_request: - branches: ["main"] + branches: [ "main" ] env: REGISTRY: ghcr.io @@ -14,8 +15,17 @@ env: jobs: # docker image build and upload to ghcr build-and-push-image: - # run this job only if the pull request is from the same repository - if: github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.body && contains(github.event.pull_request.body, '+build') + # run this job if either: + # 1. The pull request is from the same repository and contains '+build' in the body + # 2. The workflow is manually triggered (workflow_dispatch event) + if: > + ( + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.body && + contains(github.event.pull_request.body, '+build') + ) || + github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest permissions: contents: read @@ -46,7 +56,7 @@ jobs: - name: Get short SHA id: get-short-sha run: | - SHA="$(echo ${{ github.event.pull_request.head.sha }} | cut -c 1-7)" + SHA="$(echo ${GITHUB_SHA::7})" echo "sha=$SHA" >> "$GITHUB_OUTPUT" - name: Build and push Docker image From 34f09ec3fecec286ed316cb64d05660d0081991e Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Mon, 22 Apr 2024 22:04:04 +0530 Subject: [PATCH 21/60] propagate lock manager down to replicator wrapper --- libsql-server/Cargo.toml | 7 ++--- libsql-server/src/connection/libsql.rs | 1 + libsql-server/src/lib.rs | 4 ++- libsql-server/src/namespace/mod.rs | 9 ++++++- .../src/namespace/replication_wal.rs | 26 ++++++++++++++++--- libsql-storage/src/lib.rs | 20 -------------- 6 files changed, 38 insertions(+), 29 deletions(-) diff --git a/libsql-server/Cargo.toml b/libsql-server/Cargo.toml index 7de7d65fe6..6f51e3902c 100644 --- a/libsql-server/Cargo.toml +++ b/libsql-server/Cargo.toml @@ -22,7 +22,7 @@ bottomless = { version = "0", path = "../bottomless", features = ["libsql_linked bytes = { version = "1.2.1", features = ["serde"] } bytesize = { version = "1.2.0", features = ["serde"] } chrono = { version = "0.4.26", features = ["serde"] } -clap = { version = "4.0.23", features = [ "derive", "env", "string" ] } +clap = { version = "4.0.23", features = ["derive", "env", "string"] } console-subscriber = { git = "https://github.com/tokio-rs/console.git", rev = "5a80b98", optional = true } crc = "3.0.0" enclose = "1.1" @@ -62,7 +62,8 @@ sha2 = "0.10" sha256 = "1.1.3" libsql-sys = { path = "../libsql-sys", features = ["wal"], default-features = false } libsql-hrana = { path = "../libsql-hrana" } -sqlite3-parser = { package = "libsql-sqlite3-parser", path = "../vendored/sqlite3-parser", version = "0.11.0", default-features = false, features = [ "YYNOERRORRECOVERY" ] } +libsql-storage = { path = "../libsql-storage" } +sqlite3-parser = { package = "libsql-sqlite3-parser", path = "../vendored/sqlite3-parser", version = "0.11.0", default-features = false, features = ["YYNOERRORRECOVERY"] } tempfile = "3.7.0" thiserror = "1.0.38" tokio = { version = "1.22.2", features = ["rt-multi-thread", "net", "io-std", "io-util", "time", "macros", "sync", "fs", "signal"] } @@ -93,7 +94,7 @@ aws-sdk-s3 = "0.28" env_logger = "0.10" hyper = { version = "0.14", features = ["client"] } insta = { version = "1.26.0", features = ["json"] } -libsql = { path = "../libsql/"} +libsql = { path = "../libsql/" } libsql-client = { version = "0.6.5", default-features = false, features = ["reqwest_backend"] } proptest = "1.0.0" rand = "0.8.5" diff --git a/libsql-server/src/connection/libsql.rs b/libsql-server/src/connection/libsql.rs index 38892957dc..224260112b 100644 --- a/libsql-server/src/connection/libsql.rs +++ b/libsql-server/src/connection/libsql.rs @@ -314,6 +314,7 @@ where move || -> crate::Result<_> { let manager = ManagedConnectionWalWrapper::new(connection_manager); let id = manager.id(); + // let wal = libsql_storage::DurableWalManager; let wal = Sqlite3WalManager::default().wrap(manager).wrap(wal_wrapper); let conn = Connection::new( diff --git a/libsql-server/src/lib.rs b/libsql-server/src/lib.rs index 3eabaaf2b7..39518e4ed0 100644 --- a/libsql-server/src/lib.rs +++ b/libsql-server/src/lib.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; -use std::sync::{Arc, Weak}; +use std::sync::{Arc, Mutex, Weak}; use crate::connection::{Connection, MakeConnection}; use crate::database::DatabaseKind; @@ -52,6 +52,7 @@ pub mod rpc; pub mod version; pub use hrana::proto as hrana_proto; +use libsql_storage::LockManager; mod database; mod error; @@ -429,6 +430,7 @@ where channel: channel.clone(), uri: uri.clone(), migration_scheduler: scheduler_sender.into(), + lock_manager: Arc::new(Mutex::new(LockManager::new())), }; let (metastore_conn_maker, meta_store_wal_manager) = diff --git a/libsql-server/src/namespace/mod.rs b/libsql-server/src/namespace/mod.rs index 263db1b156..c54171b5e4 100644 --- a/libsql-server/src/namespace/mod.rs +++ b/libsql-server/src/namespace/mod.rs @@ -50,6 +50,7 @@ use crate::{ }; pub use fork::ForkError; +use libsql_storage::LockManager; use self::fork::{ForkTask, PointInTimeRestore}; use self::meta_store::MetaStoreHandle; @@ -347,7 +348,11 @@ impl Namespace { ) .await?; - let wal_wrapper = make_replication_wal_wrapper(bottomless_replicator, logger.clone()); + let wal_wrapper = make_replication_wal_wrapper( + bottomless_replicator, + logger.clone(), + ns_config.lock_manager.clone(), + ); let connection_maker = MakeLibSqlConn::new( db_path.to_path_buf(), wal_wrapper.clone(), @@ -736,6 +741,8 @@ pub struct NamespaceConfig { pub(crate) bottomless_replication: Option, pub(crate) scripted_backup: Option, pub(crate) migration_scheduler: SchedulerHandle, + + pub(crate) lock_manager: Arc>, } pub type DumpStream = diff --git a/libsql-server/src/namespace/replication_wal.rs b/libsql-server/src/namespace/replication_wal.rs index 70e7c7ac32..9636740d21 100644 --- a/libsql-server/src/namespace/replication_wal.rs +++ b/libsql-server/src/namespace/replication_wal.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use bottomless::bottomless_wal::BottomlessWalWrapper; use bottomless::replicator::Replicator; @@ -8,14 +8,32 @@ use crate::connection::connection_manager::ManagedConnectionWal; use crate::replication::primary::replication_logger_wal::ReplicationLoggerWalWrapper; use crate::replication::ReplicationLogger; +// pub type ReplicationWalManager = +// WalWrapper, ReplicationLoggerWalManager>; +// pub type ReplicationWal = WrappedWal, ReplicationLoggerWal>; + pub type ReplicationWalWrapper = Then, ManagedConnectionWal>; +// pub fn make_replication_wal( +// bottomless: Option, +// logger: Arc, +// lock_manager: Arc>, +// ) -> ReplicationWalManager { +// let wal_manager = libsql_storage::DurableWalManager::new(lock_manager); +// WalWrapper::new( +// bottomless.map(|b| BottomlessWalWrapper::new(Arc::new(std::sync::Mutex::new(Some(b))))), +// ReplicationLoggerWalManager::new(wal_manager, logger), +// ) +// } + pub fn make_replication_wal_wrapper( bottomless: Option, logger: Arc, + lock_manager: Arc>, ) -> ReplicationWalWrapper { - ReplicationLoggerWalWrapper::new(logger).then( - bottomless.map(|b| BottomlessWalWrapper::new(Arc::new(std::sync::Mutex::new(Some(b))))), - ) + let wal_manager = libsql_storage::DurableWalManager::new(lock_manager); + let btm = + bottomless.map(|b| BottomlessWalWrapper::new(Arc::new(std::sync::Mutex::new(Some(b))))); + ReplicationLoggerWalWrapper::new(logger).then(btm) } diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index c4508252ff..33498eb555 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -169,18 +169,6 @@ impl Wal for DurableWal { Ok(()) } - fn frame_page_no(&self, frame_no: std::num::NonZeroU32) -> Option { - trace!("DurableWal::frame_page_no(frame_no: {:?})", frame_no); - let rt = tokio::runtime::Handle::current(); - let frame_no = frame_no.get() as u64; - let req = rpc::FramePageNumReq { frame_no }; - let mut binding = self.client.lock(); - let resp = binding.frame_page_num(req); - let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); - let page_no = resp.into_inner().page_no; - std::num::NonZeroU32::new(page_no as u32) - } - fn db_size(&self) -> u32 { trace!("DurableWal::db_size() => {}", self.db_size); self.db_size @@ -300,14 +288,6 @@ impl Wal for DurableWal { trace!("DurableWal::frames_in_wal() = {}", count); count } - - fn backfilled(&self) -> u32 { - todo!() - } - - fn db_file(&self) -> &libsql_sys::wal::Sqlite3File { - todo!() - } } pub struct LockManager { From a690bf98717fe05198417631b47cfa29f488c6bc Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Tue, 23 Apr 2024 14:42:36 +0530 Subject: [PATCH 22/60] create the handle if it doesnt exist --- libsql-storage/src/lib.rs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index c4508252ff..e2745582c0 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -85,11 +85,18 @@ pub struct DurableWal { db_size: u32, name: String, lock_manager: Arc>, + rt: tokio::runtime::Handle, } impl DurableWal { fn new(lock_manager: Arc>) -> Self { - let rt = tokio::runtime::Handle::current(); + let rt = match tokio::runtime::Handle::try_current() { + Ok(h) => h, + Err(_) => { + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.handle().clone() + } + }; // connect to external storage server // export LIBSQL_STORAGE_SERVER_ADDR=http://libsql-storage-server.internal:5002 let address = std::env::var("LIBSQL_STORAGE_SERVER_ADDR") @@ -111,6 +118,7 @@ impl DurableWal { db_size, name: uuid::Uuid::new_v4().to_string(), lock_manager, + rt, } } } @@ -134,13 +142,12 @@ impl Wal for DurableWal { page_no: std::num::NonZeroU32, ) -> Result> { trace!("DurableWal::find_frame(page_no: {:?})", page_no); - let rt = tokio::runtime::Handle::current(); let req = rpc::FindFrameReq { page_no: page_no.get() as u64, }; let mut binding = self.client.lock(); let resp = binding.find_frame(req); - let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); + let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); let frame_no = resp .into_inner() .frame_no @@ -156,12 +163,11 @@ impl Wal for DurableWal { buffer.copy_from_slice(&frame); return Ok(()); } - let rt = tokio::runtime::Handle::current(); let frame_no = frame_no.get() as u64; let req = rpc::ReadFrameReq { frame_no }; let mut binding = self.client.lock(); let resp = binding.read_frame(req); - let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); + let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); let frame = resp.into_inner().frame.unwrap(); buffer.copy_from_slice(&frame); self.page_frames @@ -171,12 +177,11 @@ impl Wal for DurableWal { fn frame_page_no(&self, frame_no: std::num::NonZeroU32) -> Option { trace!("DurableWal::frame_page_no(frame_no: {:?})", frame_no); - let rt = tokio::runtime::Handle::current(); let frame_no = frame_no.get() as u64; let req = rpc::FramePageNumReq { frame_no }; let mut binding = self.client.lock(); let resp = binding.frame_page_num(req); - let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); + let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); let page_no = resp.into_inner().page_no; std::num::NonZeroU32::new(page_no as u32) } @@ -236,7 +241,6 @@ impl Wal for DurableWal { ) -> Result { trace!("name = {}", self.name); trace!("DurableWal::insert_frames(page_size: {}, size_after: {}, is_commit: {}, sync_flags: {})", page_size, size_after, is_commit, sync_flags); - let rt = tokio::runtime::Handle::current(); let frames = page_headers .iter() .map(|header| { @@ -250,7 +254,7 @@ impl Wal for DurableWal { let req = rpc::InsertFramesReq { frames }; let mut binding = self.client.lock(); let resp = binding.insert_frames(req); - let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); + let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); self.db_size = size_after; Ok(resp.into_inner().num_frames as usize) } @@ -291,11 +295,10 @@ impl Wal for DurableWal { } fn frames_in_wal(&self) -> u32 { - let rt = tokio::runtime::Handle::current(); let req = rpc::FramesInWalReq {}; let mut binding = self.client.lock(); let resp = binding.frames_in_wal(req); - let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); + let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); let count = resp.into_inner().count; trace!("DurableWal::frames_in_wal() = {}", count); count From d59868dbd3eca216e95b90dcb7765c97ad1ae523 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Tue, 23 Apr 2024 15:23:57 +0530 Subject: [PATCH 23/60] store a ref to runtime --- libsql-storage/src/lib.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index e2745582c0..989a1ab689 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -85,16 +85,18 @@ pub struct DurableWal { db_size: u32, name: String, lock_manager: Arc>, + runtime: Option, rt: tokio::runtime::Handle, } impl DurableWal { fn new(lock_manager: Arc>) -> Self { - let rt = match tokio::runtime::Handle::try_current() { - Ok(h) => h, + let (runtime, rt) = match tokio::runtime::Handle::try_current() { + Ok(h) => (None, h), Err(_) => { let rt = tokio::runtime::Runtime::new().unwrap(); - rt.handle().clone() + let handle = rt.handle().clone(); + (Some(rt), handle) } }; // connect to external storage server @@ -118,6 +120,7 @@ impl DurableWal { db_size, name: uuid::Uuid::new_v4().to_string(), lock_manager, + runtime, rt, } } From c565609a62c1338e57a6dd03cd0c0938f1e3a70d Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Tue, 23 Apr 2024 17:02:50 +0530 Subject: [PATCH 24/60] Add destroy WAL --- libsql-storage/proto/storage.proto | 5 ++ libsql-storage/src/generated/storage.rs | 73 +++++++++++++++++++++++++ libsql-storage/src/lib.rs | 9 ++- 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/libsql-storage/proto/storage.proto b/libsql-storage/proto/storage.proto index 72496d4b14..ab6ce6edf5 100644 --- a/libsql-storage/proto/storage.proto +++ b/libsql-storage/proto/storage.proto @@ -55,6 +55,10 @@ message FramePageNumResp { uint64 page_no = 1; } +message DestroyReq {} + +message DestroyResp {} + service Storage { rpc InsertFrames(InsertFramesReq) returns (InsertFramesResp) {} rpc FindFrame(FindFrameReq) returns (FindFrameResp) {} @@ -62,4 +66,5 @@ service Storage { rpc DbSize(DbSizeReq) returns (DbSizeResp) {} rpc FramesInWAL(FramesInWALReq) returns (FramesInWALResp) {} rpc FramePageNum(FramePageNumReq) returns (FramePageNumResp) {} + rpc Destroy(DestroyReq) returns (DestroyResp) {} } diff --git a/libsql-storage/src/generated/storage.rs b/libsql-storage/src/generated/storage.rs index b4ce06ffa2..2578e40f1e 100644 --- a/libsql-storage/src/generated/storage.rs +++ b/libsql-storage/src/generated/storage.rs @@ -72,6 +72,12 @@ pub struct FramePageNumResp { #[prost(uint64, tag = "1")] pub page_no: u64, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DestroyReq {} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DestroyResp {} /// Generated client implementations. pub mod storage_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] @@ -293,6 +299,25 @@ pub mod storage_client { .insert(GrpcMethod::new("storage.Storage", "FramePageNum")); self.inner.unary(req, path, codec).await } + pub async fn destroy( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/storage.Storage/Destroy"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("storage.Storage", "Destroy")); + self.inner.unary(req, path, codec).await + } } } /// Generated server implementations. @@ -332,6 +357,10 @@ pub mod storage_server { tonic::Response, tonic::Status, >; + async fn destroy( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] pub struct StorageServer { @@ -676,6 +705,50 @@ pub mod storage_server { }; Box::pin(fut) } + "/storage.Storage/Destroy" => { + #[allow(non_camel_case_types)] + struct DestroySvc(pub Arc); + impl tonic::server::UnaryService + for DestroySvc { + type Response = super::DestroyResp; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::destroy(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = DestroySvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } _ => { Box::pin(async move { Ok( diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 989a1ab689..60692e3434 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -61,7 +61,14 @@ impl WalManager for DurableWalManager { fn destroy_log(&self, vfs: &mut Vfs, db_path: &std::ffi::CStr) -> Result<()> { trace!("DurableWalManager::destroy_log()"); - // TODO: implement + let address = std::env::var("LIBSQL_STORAGE_SERVER_ADDR") + .unwrap_or("http://127.0.0.1:5002".to_string()); + let client = StorageClient::connect(address); + let rt = tokio::runtime::Runtime::new().unwrap(); + let mut client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); + let req = rpc::DestroyReq {}; + let resp = client.destroy(req); + let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); Ok(()) } From 71d12a24780647f1f3440871d15b24869ac9a22e Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Tue, 23 Apr 2024 17:20:23 +0530 Subject: [PATCH 25/60] Impl destroy --- libsql-storage-server/src/main.rs | 28 +++++++++++++++++++++++++--- libsql-storage-server/src/store.rs | 1 + 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 4c1b1fd329..f9aa5b6598 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -6,9 +6,9 @@ use bytes::Bytes; use clap::Parser; use libsql_storage::rpc::storage_server::{Storage, StorageServer}; use libsql_storage::rpc::{ - DbSizeReq, DbSizeResp, FindFrameReq, FindFrameResp, FramePageNumReq, FramePageNumResp, - FramesInWalReq, FramesInWalResp, InsertFramesReq, InsertFramesResp, ReadFrameReq, - ReadFrameResp, + DbSizeReq, DbSizeResp, DestroyReq, DestroyResp, FindFrameReq, FindFrameResp, FramePageNumReq, + FramePageNumResp, FramesInWalReq, FramesInWalResp, InsertFramesReq, InsertFramesResp, + ReadFrameReq, ReadFrameResp, }; use libsql_storage_server::version::Version; use redis::{Client, Commands, RedisResult}; @@ -92,6 +92,12 @@ impl FrameStore for InMemFrameStore { fn frames_in_wal(&self) -> u64 { self.max_frame_no } + + fn destroy(&mut self) { + self.frames.clear(); + self.pages.clear(); + self.max_frame_no = 0; + } } struct RedisFrameStore { @@ -202,6 +208,13 @@ impl FrameStore for RedisFrameStore { 0 }) } + + fn destroy(&mut self) { + // remove all the keys in redis + let mut con = self.client.get_connection().unwrap(); + // send a FLUSHALL request + let _: () = redis::cmd("FLUSHALL").query(&mut con).unwrap(); + } } struct Service { @@ -272,6 +285,15 @@ impl Storage for Service { } } + async fn destroy( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + trace!("destroy()"); + self.store.lock().unwrap().destroy(); + Ok(Response::new(DestroyResp {})) + } + async fn db_size( &self, request: tonic::Request, diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index 44c02a4aac..5f1ed85b8e 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -4,4 +4,5 @@ pub trait FrameStore { fn find_frame(&self, page_no: u64) -> Option; fn frame_page_no(&self, frame_no: u64) -> Option; fn frames_in_wal(&self) -> u64; + fn destroy(&mut self); } From e3770ab05e2245f27b88cf010bef5c4409d45638 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Wed, 1 May 2024 11:18:51 +0530 Subject: [PATCH 26/60] merge and add changes --- Cargo.lock | 2 +- libsql-server/src/connection/libsql.rs | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 257a0ae3d7..27e4c2ffb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5759,7 +5759,7 @@ dependencies = [ "percent-encoding", "pin-project", "prost", - "rustls 0.21.11", + "rustls 0.21.12", "rustls-pemfile 1.0.4", "tokio", "tokio-rustls 0.24.1", diff --git a/libsql-server/src/connection/libsql.rs b/libsql-server/src/connection/libsql.rs index 046a3512b4..4ae1100723 100644 --- a/libsql-server/src/connection/libsql.rs +++ b/libsql-server/src/connection/libsql.rs @@ -4,6 +4,7 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use libsql_storage::{DurableWalManager, LockManager}; use libsql_sys::wal::wrapper::{WrapWal, WrappedWal}; use libsql_sys::wal::{BusyHandler, CheckpointCallback, Sqlite3WalManager, Wal, WalManager}; use libsql_sys::EncryptionConfig; @@ -48,6 +49,7 @@ pub struct MakeLibSqlConn { encryption_config: Option, block_writes: Arc, resolve_attach_path: ResolveNamespacePathFn, + lock_manager: Arc>, } impl MakeLibSqlConn @@ -68,6 +70,7 @@ where encryption_config: Option, block_writes: Arc, resolve_attach_path: ResolveNamespacePathFn, + lock_manager: Arc>, ) -> Result { let mut this = Self { db_path, @@ -83,7 +86,8 @@ where encryption_config, block_writes, resolve_attach_path, - connection_manager: ConnectionManager::default(), + connection_manager: ConnectionManager::new(Default::default()), + lock_manager, }; let db = this.try_create_db().await?; @@ -138,6 +142,7 @@ where self.block_writes.clone(), self.resolve_attach_path.clone(), self.connection_manager.clone(), + self.lock_manager.clone(), ) .await } @@ -307,13 +312,15 @@ where block_writes: Arc, resolve_attach_path: ResolveNamespacePathFn, connection_manager: ConnectionManager, + lock_manager: Arc>, ) -> crate::Result { let (conn, id) = tokio::task::spawn_blocking({ let connection_manager = connection_manager.clone(); move || -> crate::Result<_> { let manager = ManagedConnectionWalWrapper::new(connection_manager); let id = manager.id(); - let wal = Sqlite3WalManager::default().wrap(manager).wrap(wal_wrapper); + let durable_wal = DurableWalManager::new(lock_manager); + let wal = durable_wal.wrap(manager).wrap(wal_wrapper); let conn = Connection::new( path.as_ref(), From a19e31b1cbce325d7a912152015497656d233899 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Wed, 1 May 2024 11:33:37 +0530 Subject: [PATCH 27/60] merge hell --- libsql-server/src/connection/libsql.rs | 8 +++--- libsql-server/src/namespace/mod.rs | 6 +---- .../src/namespace/replication_wal.rs | 26 +++---------------- 3 files changed, 10 insertions(+), 30 deletions(-) diff --git a/libsql-server/src/connection/libsql.rs b/libsql-server/src/connection/libsql.rs index 4ae1100723..86e3b93b14 100644 --- a/libsql-server/src/connection/libsql.rs +++ b/libsql-server/src/connection/libsql.rs @@ -31,7 +31,7 @@ use super::connection_manager::{ use super::program::{ check_describe_auth, check_program_auth, DescribeCol, DescribeParam, DescribeResponse, Vm, }; -use super::{MakeConnection, Program, RequestContext}; +use super::{MakeConnection, Program, RequestContext, TXN_TIMEOUT}; pub struct MakeLibSqlConn { db_path: PathBuf, @@ -72,6 +72,8 @@ where resolve_attach_path: ResolveNamespacePathFn, lock_manager: Arc>, ) -> Result { + let txn_timeout = config_store.get().txn_timeout.unwrap_or(TXN_TIMEOUT); + let mut this = Self { db_path, stats, @@ -86,7 +88,7 @@ where encryption_config, block_writes, resolve_attach_path, - connection_manager: ConnectionManager::new(Default::default()), + connection_manager: ConnectionManager::new(txn_timeout), lock_manager, }; @@ -177,7 +179,7 @@ impl LibSqlConnection { tokio::sync::watch::channel(None).1, Default::default(), Arc::new(|_| unreachable!()), - ConnectionManager::default(), + ConnectionManager::new(TXN_TIMEOUT), ) .await .unwrap() diff --git a/libsql-server/src/namespace/mod.rs b/libsql-server/src/namespace/mod.rs index 3902ebd167..681b394d85 100644 --- a/libsql-server/src/namespace/mod.rs +++ b/libsql-server/src/namespace/mod.rs @@ -348,11 +348,7 @@ impl Namespace { ) .await?; - let wal_wrapper = make_replication_wal_wrapper( - bottomless_replicator, - logger.clone(), - ns_config.lock_manager.clone(), - ); + let wal_wrapper = make_replication_wal_wrapper(bottomless_replicator, logger.clone()); let connection_maker = MakeLibSqlConn::new( db_path.to_path_buf(), wal_wrapper.clone(), diff --git a/libsql-server/src/namespace/replication_wal.rs b/libsql-server/src/namespace/replication_wal.rs index 9636740d21..70e7c7ac32 100644 --- a/libsql-server/src/namespace/replication_wal.rs +++ b/libsql-server/src/namespace/replication_wal.rs @@ -1,4 +1,4 @@ -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use bottomless::bottomless_wal::BottomlessWalWrapper; use bottomless::replicator::Replicator; @@ -8,32 +8,14 @@ use crate::connection::connection_manager::ManagedConnectionWal; use crate::replication::primary::replication_logger_wal::ReplicationLoggerWalWrapper; use crate::replication::ReplicationLogger; -// pub type ReplicationWalManager = -// WalWrapper, ReplicationLoggerWalManager>; -// pub type ReplicationWal = WrappedWal, ReplicationLoggerWal>; - pub type ReplicationWalWrapper = Then, ManagedConnectionWal>; -// pub fn make_replication_wal( -// bottomless: Option, -// logger: Arc, -// lock_manager: Arc>, -// ) -> ReplicationWalManager { -// let wal_manager = libsql_storage::DurableWalManager::new(lock_manager); -// WalWrapper::new( -// bottomless.map(|b| BottomlessWalWrapper::new(Arc::new(std::sync::Mutex::new(Some(b))))), -// ReplicationLoggerWalManager::new(wal_manager, logger), -// ) -// } - pub fn make_replication_wal_wrapper( bottomless: Option, logger: Arc, - lock_manager: Arc>, ) -> ReplicationWalWrapper { - let wal_manager = libsql_storage::DurableWalManager::new(lock_manager); - let btm = - bottomless.map(|b| BottomlessWalWrapper::new(Arc::new(std::sync::Mutex::new(Some(b))))); - ReplicationLoggerWalWrapper::new(logger).then(btm) + ReplicationLoggerWalWrapper::new(logger).then( + bottomless.map(|b| BottomlessWalWrapper::new(Arc::new(std::sync::Mutex::new(Some(b))))), + ) } From 7d039b94fed72ee84164df23284347860e47163e Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Wed, 1 May 2024 15:58:59 +0530 Subject: [PATCH 28/60] remove todo --- libsql-storage/src/lib.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index b43bd1444c..7cb752e0f4 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -222,12 +222,10 @@ impl Wal for DurableWal { Ok(()) } - fn savepoint(&mut self, rollback_data: &mut [u32]) { - todo!() - } + fn savepoint(&mut self, rollback_data: &mut [u32]) {} fn savepoint_undo(&mut self, rollback_data: &mut [u32]) -> Result<()> { - todo!() + Ok(()) } fn insert_frames( From 1515b4eba30034cba119e17d0bd3c6c1a3c83187 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Wed, 1 May 2024 22:17:48 +0530 Subject: [PATCH 29/60] missing work and minor fixes --- libsql-storage/src/lib.rs | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 7cb752e0f4..e45691649f 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -61,21 +61,21 @@ impl WalManager for DurableWalManager { fn destroy_log(&self, vfs: &mut Vfs, db_path: &std::ffi::CStr) -> Result<()> { trace!("DurableWalManager::destroy_log()"); - let address = std::env::var("LIBSQL_STORAGE_SERVER_ADDR") - .unwrap_or("http://127.0.0.1:5002".to_string()); - let client = StorageClient::connect(address); - let rt = tokio::runtime::Runtime::new().unwrap(); - let mut client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); - let req = rpc::DestroyReq {}; - let resp = client.destroy(req); - let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); + // let address = std::env::var("LIBSQL_STORAGE_SERVER_ADDR") + // .unwrap_or("http://127.0.0.1:5002".to_string()); + // let client = StorageClient::connect(address); + // let rt = tokio::runtime::Runtime::new().unwrap(); + // let mut client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); + // let req = rpc::DestroyReq {}; + // let resp = client.destroy(req); + // let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); Ok(()) } fn log_exists(&self, vfs: &mut Vfs, db_path: &std::ffi::CStr) -> Result { trace!("DurableWalManager::log_exists()"); // TODO: implement - Ok(false) + Ok(true) } fn destroy(self) @@ -140,11 +140,14 @@ impl Wal for DurableWal { fn begin_read_txn(&mut self) -> Result { trace!("DurableWal::begin_read_txn()"); + // TODO: give a read lock for this conn + // note down max frame number Ok(true) } fn end_read_txn(&mut self) { trace!("DurableWal::end_read_txn()"); + // TODO: drop both read or write lock } fn find_frame( @@ -152,6 +155,7 @@ impl Wal for DurableWal { page_no: std::num::NonZeroU32, ) -> Result> { trace!("DurableWal::find_frame(page_no: {:?})", page_no); + // TODO: send max frame number in the request let req = rpc::FindFrameReq { page_no: page_no.get() as u64, }; @@ -191,6 +195,8 @@ impl Wal for DurableWal { } fn begin_write_txn(&mut self) -> Result<()> { + // todo: check if the connection holds a read lock + // then try to acquire a write lock let mut lock_manager = self.lock_manager.lock().unwrap(); if !lock_manager.lock("default".to_string(), self.name.clone()) { trace!( @@ -207,7 +213,7 @@ impl Wal for DurableWal { } fn end_write_txn(&mut self) -> Result<()> { - // release lock + // release only if lock is write lock let mut lock_manager = self.lock_manager.lock().unwrap(); trace!( "DurableWal::end_write_txn() id = {}, unlocked = {}", @@ -236,6 +242,8 @@ impl Wal for DurableWal { is_commit: bool, sync_flags: std::ffi::c_int, ) -> Result { + // TODO: check if it has a write lock + // check if the size_after is > 0, if so then mark txn as committed trace!("name = {}", self.name); trace!("DurableWal::insert_frames(page_size: {}, size_after: {}, is_commit: {}, sync_flags: {})", page_size, size_after, is_commit, sync_flags); let frames = page_headers @@ -279,12 +287,10 @@ impl Wal for DurableWal { fn uses_heap_memory(&self) -> bool { trace!("DurableWal::uses_heap_memory()"); - false + true } - fn set_db(&mut self, db: &mut libsql_sys::wal::Sqlite3Db) { - todo!() - } + fn set_db(&mut self, db: &mut libsql_sys::wal::Sqlite3Db) {} fn callback(&self) -> i32 { trace!("DurableWal::callback()"); From fa75ca3b6fa235e97047f60ab6adc5933504c253 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 2 May 2024 19:13:59 +0530 Subject: [PATCH 30/60] minor diff --- libsql-storage/src/lib.rs | 116 +++++++++++++++++++++++++++----------- 1 file changed, 83 insertions(+), 33 deletions(-) diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index e45691649f..152debe4fd 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -1,9 +1,9 @@ -// use libsql_sys::ffi::SQLITE_BUSY; -// use libsql_sys::rusqlite; use libsql_sys::ffi::SQLITE_BUSY; use libsql_sys::rusqlite; use libsql_sys::wal::{Result, Vfs, Wal, WalManager}; use sieve_cache::SieveCache; +use std::collections::BTreeMap; +use std::mem::size_of; use std::sync::{Arc, Mutex}; use tonic::transport::Channel; use tracing::trace; @@ -94,6 +94,7 @@ pub struct DurableWal { lock_manager: Arc>, runtime: Option, rt: tokio::runtime::Handle, + write_cache: BTreeMap, } impl DurableWal { @@ -129,8 +130,39 @@ impl DurableWal { lock_manager, runtime, rt, + write_cache: BTreeMap::new(), } } + + fn find_frame_by_page_no( + &mut self, + page_no: std::num::NonZeroU32, + ) -> Result> { + trace!("DurableWal::find_frame_by_page_no(page_no: {:?})", page_no); + // TODO: send max frame number in the request + let req = rpc::FindFrameReq { + page_no: page_no.get() as u64, + }; + let mut binding = self.client.lock(); + let resp = binding.find_frame(req); + let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); + let frame_no = resp + .into_inner() + .frame_no + .map(|page_no| std::num::NonZeroU32::new(page_no as u32)) + .flatten(); + Ok(frame_no) + } + + fn frames_count(&self) -> u32 { + let req = rpc::FramesInWalReq {}; + let mut binding = self.client.lock(); + let resp = binding.frames_in_wal(req); + let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); + let count = resp.into_inner().count; + trace!("DurableWal::frames_in_wal() = {}", count); + count + } } impl Wal for DurableWal { @@ -155,25 +187,31 @@ impl Wal for DurableWal { page_no: std::num::NonZeroU32, ) -> Result> { trace!("DurableWal::find_frame(page_no: {:?})", page_no); - // TODO: send max frame number in the request - let req = rpc::FindFrameReq { - page_no: page_no.get() as u64, - }; - let mut binding = self.client.lock(); - let resp = binding.find_frame(req); - let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); - let frame_no = resp - .into_inner() - .frame_no - .map(|page_no| std::num::NonZeroU32::new(page_no as u32)) - .flatten(); - Ok(frame_no) + let frame_no = self.find_frame_by_page_no(page_no).unwrap(); + if frame_no.is_none() { + return Ok(None); + } + return Ok(Some(page_no)); } - fn read_frame(&mut self, frame_no: std::num::NonZeroU32, buffer: &mut [u8]) -> Result<()> { - trace!("DurableWal::read_frame(frame_no: {:?})", frame_no); + // read_frame reads the page, not the frame + fn read_frame(&mut self, page_no: std::num::NonZeroU32, buffer: &mut [u8]) -> Result<()> { + trace!("DurableWal::read_frame(page_no: {:?})", page_no); + if let Some(frame) = self.write_cache.get(&(u32::from(page_no))) { + trace!( + "DurableWal::read_frame(page_no: {:?}) -- write cache hit", + page_no + ); + buffer.copy_from_slice(&frame.data); + return Ok(()); + } + let frame_no = self.find_frame_by_page_no(page_no).unwrap().unwrap(); // check if the frame exists in the local cache if let Some(frame) = self.page_frames.get(&frame_no) { + trace!( + "DurableWal::read_frame(page_no: {:?}) -- read cache hit", + page_no + ); buffer.copy_from_slice(&frame); return Ok(()); } @@ -191,7 +229,7 @@ impl Wal for DurableWal { fn db_size(&self) -> u32 { trace!("DurableWal::db_size() => {}", self.db_size); - self.db_size + self.frames_count() } fn begin_write_txn(&mut self) -> Result<()> { @@ -246,17 +284,28 @@ impl Wal for DurableWal { // check if the size_after is > 0, if so then mark txn as committed trace!("name = {}", self.name); trace!("DurableWal::insert_frames(page_size: {}, size_after: {}, is_commit: {}, sync_flags: {})", page_size, size_after, is_commit, sync_flags); - let frames = page_headers - .iter() - .map(|header| { - let (page_no, frame) = header; + // add data from frame_headers to writeCache + for (page_no, frame) in page_headers.iter() { + self.write_cache.insert( + page_no, rpc::Frame { page_no: page_no as u64, data: frame.to_vec(), - } - }) - .collect(); - let req = rpc::InsertFramesReq { frames }; + }, + ); + // todo: update size after + } + + if size_after <= 0 { + // todo: update new size + return Ok(0); + } + + let req = rpc::InsertFramesReq { + frames: self.write_cache.values().cloned().collect(), + }; + + self.write_cache.clear(); let mut binding = self.client.lock(); let resp = binding.insert_frames(req); let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); @@ -298,13 +347,14 @@ impl Wal for DurableWal { } fn frames_in_wal(&self) -> u32 { - let req = rpc::FramesInWalReq {}; - let mut binding = self.client.lock(); - let resp = binding.frames_in_wal(req); - let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); - let count = resp.into_inner().count; - trace!("DurableWal::frames_in_wal() = {}", count); - count + // let req = rpc::FramesInWalReq {}; + // let mut binding = self.client.lock(); + // let resp = binding.frames_in_wal(req); + // let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); + // let count = resp.into_inner().count; + // trace!("DurableWal::frames_in_wal() = {}", count); + // count + 0 } } From 9245b63e9c6247ca3e24b767bfbbc50127246f5c Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 2 May 2024 22:51:22 +0530 Subject: [PATCH 31/60] set limits on client, read all payload --- libsql-storage-server/src/main.rs | 20 ++++++++++++++++++-- libsql-storage/src/lib.rs | 17 +++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index f9aa5b6598..326f938df0 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -56,7 +56,7 @@ impl InMemFrameStore { impl FrameStore for InMemFrameStore { // inserts a new frame for the page number and returns the new frame value - fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64 { + fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { let frame_no = self.max_frame_no + 1; self.max_frame_no = frame_no; self.frames.insert( @@ -240,7 +240,23 @@ impl Storage for Service { trace!("insert_frames()"); let mut num_frames = 0; let mut store = self.store.lock().unwrap(); - for frame in request.into_inner().frames { + trace!("insert_frames() got lock"); + let frames = request + .into_inner() + .frames + .into_iter() + .map(|frame| FrameData { + page_no: frame.page_no, + data: frame.data.into(), + }); + let all_data: Vec = frames + .clone() + .map(|f| f.data.clone().to_vec()) + .flatten() + .collect(); + trace!("insert_frames() got frames (bytes): {:?}", all_data.len()); + trace!("insert_frames() got frames: {:?}", frames.len()); + for frame in frames { trace!( "inserted for page {} frame {}", frame.page_no, diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 152debe4fd..9f37a5657c 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -112,7 +112,16 @@ impl DurableWal { let address = std::env::var("LIBSQL_STORAGE_SERVER_ADDR") .unwrap_or("http://127.0.0.1:5002".to_string()); trace!("DurableWal::new() address = {}", address); + + // let channel = Channel::from_static(address).connect_lazy().unwrap(); + // let channel = Channel::builder("127.0.0.1:3000".parse().unwrap()).connect_lazy(); + let client = StorageClient::connect(address); + // let client = client.max_encoding_message_size(100); + // let client = StorageClient::new(channel) + // .max_encoding_message_size(100) + // .unwrap(); + let mut client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); let req = rpc::DbSizeReq {}; @@ -304,10 +313,14 @@ impl Wal for DurableWal { let req = rpc::InsertFramesReq { frames: self.write_cache.values().cloned().collect(), }; - self.write_cache.clear(); let mut binding = self.client.lock(); - let resp = binding.insert_frames(req); + trace!("sending DurableWal::insert_frames() {:?}", req.frames.len()); + let mut c = binding + .clone() + .max_encoding_message_size(256 * 1024 * 1024) + .max_decoding_message_size(256 * 1024 * 1024); + let resp = c.insert_frames(req); let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); self.db_size = size_after; Ok(resp.into_inner().num_frames as usize) From 8ba9dd4b05be8f09c2c9e2e4abd49f47e94074cb Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Wed, 15 May 2024 19:33:52 +0530 Subject: [PATCH 32/60] refactor --- libsql-storage-server/src/main.rs | 185 +--------------------- libsql-storage-server/src/memory_store.rs | 65 ++++++++ libsql-storage-server/src/redis_store.rs | 121 ++++++++++++++ 3 files changed, 189 insertions(+), 182 deletions(-) create mode 100644 libsql-storage-server/src/memory_store.rs create mode 100644 libsql-storage-server/src/redis_store.rs diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 326f938df0..f711af7f14 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -1,3 +1,5 @@ +mod memory_store; +mod redis_store; mod store; use crate::store::FrameStore; @@ -12,9 +14,7 @@ use libsql_storage::rpc::{ }; use libsql_storage_server::version::Version; use redis::{Client, Commands, RedisResult}; -use std::collections::BTreeMap; -use std::fmt::format; -use std::iter::Map; +use redis_store::RedisFrameStore; use std::net::SocketAddr; use std::sync::atomic::AtomicU32; use std::sync::{Arc, Mutex}; @@ -33,190 +33,11 @@ struct Cli { } #[derive(Default)] - struct FrameData { page_no: u64, data: bytes::Bytes, } -#[derive(Default)] -struct InMemFrameStore { - // contains a frame data, key is the frame number - frames: BTreeMap, - // pages map contains the page number as a key and the list of frames for the page as a value - pages: BTreeMap>, - max_frame_no: u64, -} - -impl InMemFrameStore { - pub fn new() -> Self { - Self::default() - } -} - -impl FrameStore for InMemFrameStore { - // inserts a new frame for the page number and returns the new frame value - fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { - let frame_no = self.max_frame_no + 1; - self.max_frame_no = frame_no; - self.frames.insert( - frame_no, - FrameData { - page_no, - data: frame, - }, - ); - self.pages - .entry(page_no) - .or_insert_with(Vec::new) - .push(frame_no); - frame_no - } - - fn read_frame(&self, frame_no: u64) -> Option { - self.frames.get(&frame_no).map(|frame| frame.data.clone()) - } - - // given a page number, return the maximum frame for the page - fn find_frame(&self, page_no: u64) -> Option { - self.pages - .get(&page_no) - .map(|frames| *frames.last().unwrap()) - } - - // given a frame num, return the page number - fn frame_page_no(&self, frame_no: u64) -> Option { - self.frames.get(&frame_no).map(|frame| frame.page_no) - } - - fn frames_in_wal(&self) -> u64 { - self.max_frame_no - } - - fn destroy(&mut self) { - self.frames.clear(); - self.pages.clear(); - self.max_frame_no = 0; - } -} - -struct RedisFrameStore { - client: Client, -} - -impl RedisFrameStore { - pub fn new(client: Client) -> Self { - Self { client } - } -} - -impl FrameStore for RedisFrameStore { - fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { - let namespace = "default"; - let max_frame_key = format!("{}/max_frame_no", namespace); - - let mut con = self.client.get_connection().unwrap(); - // max_frame_key might change if another client inserts a frame, so do - // all this in a transaction! - let (max_frame_no,): (u64,) = - redis::transaction(&mut con, &[&max_frame_key], |con, pipe| { - let result: RedisResult = con.get(max_frame_key.clone()); - if result.is_err() && !is_nil_response(result.as_ref().err().unwrap()) { - return Err(result.err().unwrap()); - } - let max_frame_no = result.unwrap_or(0) + 1; - let frame_key = format!("f/{}/{}", namespace, max_frame_no); - let page_key = format!("p/{}/{}", namespace, page_no); - - pipe.hset::>(frame_key.clone(), "f", frame.to_vec()) - .ignore() - .hset::(frame_key.clone(), "p", page_no) - .ignore() - .set::(page_key, max_frame_no) - .ignore() - .set::(max_frame_key.clone(), max_frame_no) - .ignore() - .get(max_frame_key.clone()) - .query(con) - }) - .unwrap(); - max_frame_no - } - - fn read_frame(&self, frame_no: u64) -> Option { - let namespace = "default"; - let frame_key = format!("f/{}/{}", namespace, frame_no); - let mut con = self.client.get_connection().unwrap(); - let result = con.hget::>(frame_key.clone(), "f"); - match result { - Ok(frame) => Some(Bytes::from(frame)), - Err(e) => { - if !is_nil_response(&e) { - error!( - "read_frame() failed for frame_no={} with err={}", - frame_no, e - ); - } - None - } - } - } - - fn find_frame(&self, page_no: u64) -> Option { - let page_key = format!("p/{}/{}", "default", page_no); - let mut con = self.client.get_connection().unwrap(); - let frame_no = con.get::(page_key.clone()); - match frame_no { - Ok(frame_no) => Some(frame_no), - Err(e) => { - if !is_nil_response(&e) { - error!("find_frame() failed for page_no={} with err={}", page_no, e); - } - None - } - } - } - - fn frame_page_no(&self, frame_no: u64) -> Option { - let namespace = "default"; - let frame_key = format!("f/{}/{}", namespace, frame_no); - let mut con = self.client.get_connection().unwrap(); - let result = con.hget::(frame_key.clone(), "p"); - match result { - Ok(page_no) => Some(page_no), - Err(e) => { - if !is_nil_response(&e) { - error!( - "frame_page_no() failed for frame_no={} with err={}", - frame_no, e - ); - } - None - } - } - } - - fn frames_in_wal(&self) -> u64 { - let namespace = "default"; - let max_frame_key = format!("{}/max_frame_no", namespace); - let mut con = self.client.get_connection().unwrap(); - let result = con.get::(max_frame_key.clone()); - result.unwrap_or_else(|e| { - if !is_nil_response(&e) { - error!("frames_in_wal() failed with err={}", e); - } - 0 - }) - } - - fn destroy(&mut self) { - // remove all the keys in redis - let mut con = self.client.get_connection().unwrap(); - // send a FLUSHALL request - let _: () = redis::cmd("FLUSHALL").query(&mut con).unwrap(); - } -} - struct Service { store: Arc>, db_size: AtomicU32, diff --git a/libsql-storage-server/src/memory_store.rs b/libsql-storage-server/src/memory_store.rs new file mode 100644 index 0000000000..cd1c2ac9fd --- /dev/null +++ b/libsql-storage-server/src/memory_store.rs @@ -0,0 +1,65 @@ +use crate::store::FrameStore; +use crate::FrameData; +use bytes::Bytes; +use std::collections::BTreeMap; + +#[derive(Default)] +struct InMemFrameStore { + // contains a frame data, key is the frame number + frames: BTreeMap, + // pages map contains the page number as a key and the list of frames for the page as a value + pages: BTreeMap>, + max_frame_no: u64, +} + +impl InMemFrameStore { + pub fn new() -> Self { + Self::default() + } +} + +impl FrameStore for InMemFrameStore { + // inserts a new frame for the page number and returns the new frame value + fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { + let frame_no = self.max_frame_no + 1; + self.max_frame_no = frame_no; + self.frames.insert( + frame_no, + FrameData { + page_no, + data: frame, + }, + ); + self.pages + .entry(page_no) + .or_insert_with(Vec::new) + .push(frame_no); + frame_no + } + + fn read_frame(&self, frame_no: u64) -> Option { + self.frames.get(&frame_no).map(|frame| frame.data.clone()) + } + + // given a page number, return the maximum frame for the page + fn find_frame(&self, page_no: u64) -> Option { + self.pages + .get(&page_no) + .map(|frames| *frames.last().unwrap()) + } + + // given a frame num, return the page number + fn frame_page_no(&self, frame_no: u64) -> Option { + self.frames.get(&frame_no).map(|frame| frame.page_no) + } + + fn frames_in_wal(&self) -> u64 { + self.max_frame_no + } + + fn destroy(&mut self) { + self.frames.clear(); + self.pages.clear(); + self.max_frame_no = 0; + } +} diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs new file mode 100644 index 0000000000..a10e474e26 --- /dev/null +++ b/libsql-storage-server/src/redis_store.rs @@ -0,0 +1,121 @@ +use crate::store::FrameStore; +use redis::Client; +use redis::Commands; +use tracing::error; + +pub struct RedisFrameStore { + client: Client, +} + +impl RedisFrameStore { + pub fn new(client: Client) -> Self { + Self { client } + } +} + +impl FrameStore for RedisFrameStore { + fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { + let namespace = "default"; + let max_frame_key = format!("{}/max_frame_no", namespace); + + let mut con = self.client.get_connection().unwrap(); + // max_frame_key might change if another client inserts a frame, so do + // all this in a transaction! + let (max_frame_no,): (u64,) = + redis::transaction(&mut con, &[&max_frame_key], |con, pipe| { + let result: RedisResult = con.get(max_frame_key.clone()); + if result.is_err() && !crate::is_nil_response(result.as_ref().err().unwrap()) { + return Err(result.err().unwrap()); + } + let max_frame_no = result.unwrap_or(0) + 1; + let frame_key = format!("f/{}/{}", namespace, max_frame_no); + let page_key = format!("p/{}/{}", namespace, page_no); + + pipe.hset::>(frame_key.clone(), "f", frame.to_vec()) + .ignore() + .hset::(frame_key.clone(), "p", page_no) + .ignore() + .set::(page_key, max_frame_no) + .ignore() + .set::(max_frame_key.clone(), max_frame_no) + .ignore() + .get(max_frame_key.clone()) + .query(con) + }) + .unwrap(); + max_frame_no + } + + fn read_frame(&self, frame_no: u64) -> Option { + let namespace = "default"; + let frame_key = format!("f/{}/{}", namespace, frame_no); + let mut con = self.client.get_connection().unwrap(); + let result = con.hget::>(frame_key.clone(), "f"); + match result { + Ok(frame) => Some(Bytes::from(frame)), + Err(e) => { + if !crate::is_nil_response(&e) { + error!( + "read_frame() failed for frame_no={} with err={}", + frame_no, e + ); + } + None + } + } + } + + fn find_frame(&self, page_no: u64) -> Option { + let page_key = format!("p/{}/{}", "default", page_no); + let mut con = self.client.get_connection().unwrap(); + let frame_no = con.get::(page_key.clone()); + match frame_no { + Ok(frame_no) => Some(frame_no), + Err(e) => { + if !crate::is_nil_response(&e) { + error!("find_frame() failed for page_no={} with err={}", page_no, e); + } + None + } + } + } + + fn frame_page_no(&self, frame_no: u64) -> Option { + let namespace = "default"; + let frame_key = format!("f/{}/{}", namespace, frame_no); + let mut con = self.client.get_connection().unwrap(); + let result = con.hget::(frame_key.clone(), "p"); + match result { + Ok(page_no) => Some(page_no), + Err(e) => { + if !crate::is_nil_response(&e) { + error!( + "frame_page_no() failed for frame_no={} with err={}", + frame_no, e + ); + } + None + } + } + } + + fn frames_in_wal(&self) -> u64 { + let namespace = "default"; + let max_frame_key = format!("{}/max_frame_no", namespace); + let mut con = self.client.get_connection().unwrap(); + let result = con.get::(max_frame_key.clone()); + result.unwrap_or_else(|e| { + if !crate::is_nil_response(&e) { + error!("frames_in_wal() failed with err={}", e); + } + 0 + }) + } + + fn destroy(&mut self) { + // remove all the keys in redis + let mut con = self.client.get_connection().unwrap(); + // send a FLUSHALL request + let _: () = redis::cmd("FLUSHALL").query(&mut con).unwrap(); + } +} From a73d67e19d5aeeee35518aa87e51b9ff31cc9a9c Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 16 May 2024 21:20:36 +0530 Subject: [PATCH 33/60] Make code async and move to FDB --- Cargo.lock | 106 +++++++++++++++++++- libsql-storage-server/Cargo.toml | 2 + libsql-storage-server/src/fdb_store.rs | 122 +++++++++++++++++++++++ libsql-storage-server/src/main.rs | 36 ++++--- libsql-storage-server/src/redis_store.rs | 4 +- libsql-storage-server/src/store.rs | 12 +-- 6 files changed, 254 insertions(+), 28 deletions(-) create mode 100644 libsql-storage-server/src/fdb_store.rs diff --git a/Cargo.lock b/Cargo.lock index 27e4c2ffb6..8120e892fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1219,6 +1219,29 @@ dependencies = [ "which", ] +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags 2.5.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.60", + "which", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -2418,6 +2441,58 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foundationdb" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bf4ae7238dbdb1ff01e9f981db028515cf66883c461e29faedfea130b2728" +dependencies = [ + "async-recursion", + "async-trait", + "foundationdb-gen", + "foundationdb-macros", + "foundationdb-sys", + "futures", + "memchr", + "rand", + "serde", + "serde_bytes", + "serde_json", + "static_assertions", + "uuid", +] + +[[package]] +name = "foundationdb-gen" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36878d54a76a48e794d0fe89be2096ab5968b071e7ec25f7becfe7846f55fa77" +dependencies = [ + "xml-rs", +] + +[[package]] +name = "foundationdb-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8db6653cbc621a3810d95d55bd342be3e71181d6df21a4eb29ef986202d3f9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", + "try_map", +] + +[[package]] +name = "foundationdb-sys" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace2f49db8614b7d7e3b656a12e0059b5fbd0a4da3410b1797374bec3db269fa" +dependencies = [ + "bindgen 0.69.4", + "libc", +] + [[package]] name = "fs-set-times" version = "0.19.2" @@ -3301,7 +3376,7 @@ dependencies = [ name = "libsql-ffi" version = "0.2.1" dependencies = [ - "bindgen", + "bindgen 0.66.1", "cc", "libsql-wasmtime-bindings", ] @@ -3477,6 +3552,8 @@ dependencies = [ "anyhow", "bytes", "clap 4.5.4", + "foundationdb", + "futures", "libsql-storage", "redis", "tokio", @@ -5128,6 +5205,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "serde_bytes" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.199" @@ -5369,6 +5455,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "str_stack" version = "0.1.0" @@ -6015,6 +6107,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "try_map" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1626d07cb5c1bb2cf17d94c0be4852e8a7c02b041acec9a8c5bdda99f9d580" + [[package]] name = "tungstenite" version = "0.20.1" @@ -7070,6 +7168,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "xml-rs" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/libsql-storage-server/Cargo.toml b/libsql-storage-server/Cargo.toml index 9c92eba1eb..7eca66e379 100644 --- a/libsql-storage-server/Cargo.toml +++ b/libsql-storage-server/Cargo.toml @@ -12,6 +12,8 @@ path = "src/main.rs" anyhow = "1.0.66" bytes = "1.5.0" clap = { version = "4.0.23", features = [ "derive", "env", "string" ] } +foundationdb = { version = "0.9.0", features = ["embedded-fdb-include", "fdb-7_3"] } +futures = "0.3.30" libsql-storage = { path = "../libsql-storage" } redis = "0.25.3" tokio = { version = "1.22.2", features = ["rt-multi-thread", "net", "io-std", "io-util", "time", "macros", "sync", "fs", "signal"] } diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs new file mode 100644 index 0000000000..2f505b3330 --- /dev/null +++ b/libsql-storage-server/src/fdb_store.rs @@ -0,0 +1,122 @@ +use crate::store::FrameStore; +use bytes::Bytes; +use foundationdb::api::NetworkAutoStop; +use foundationdb::tuple::pack; +use foundationdb::tuple::unpack; +use foundationdb::Transaction; +use tracing::error; + +pub struct FDBFrameStore { + network: NetworkAutoStop, +} + +impl FDBFrameStore { + pub fn new() -> Self { + let network = unsafe { foundationdb::boot() }; + Self { network } + } + + async fn get_max_frame_no(&self, txn: &Transaction, namespace: &str) -> u64 { + let namespace = "default"; + let max_frame_key = format!("{}/max_frame_no", namespace); + let result = txn.get(&max_frame_key.as_bytes(), false).await; + if let Err(e) = result { + error!("get failed: {:?}", e); + return 0; + } + if let Ok(None) = result { + error!("page not found"); + return 0; + } + let frame_no: u64 = unpack(&result.unwrap().unwrap()).expect("failed to decode u64"); + tracing::info!("max_frame_no={}", frame_no); + frame_no + } +} + +impl FrameStore for FDBFrameStore { + async fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { + let namespace = "default"; + let max_frame_key = format!("{}/max_frame_no", namespace); + let db = foundationdb::Database::default().unwrap(); + let txn = db.create_trx().expect("unable to create transaction"); + let frame_no = self.get_max_frame_no(&txn, namespace).await; + + let frame_key = format!("{}/f/{}/f", namespace, frame_no); + let frame_page_key = format!("{}/f/{}/p", namespace, frame_no); + let page_key = format!("{}/p/{}", namespace, page_no); + + txn.set(&frame_key.as_bytes(), &frame); + txn.set(&frame_page_key.as_bytes(), &pack(&page_no)); + txn.set(&page_key.as_bytes(), &pack(&frame_no)); + txn.set(&max_frame_key.as_bytes(), &pack(&(frame_no + 1))); + txn.commit().await.expect("commit failed"); + frame_no + 1 + } + + async fn read_frame(&self, frame_no: u64) -> Option { + let namespace = "default"; + let frame_key = format!("{}/f/{}/f", namespace, frame_no); + + let db = foundationdb::Database::default().unwrap(); + let txn = db.create_trx().expect("unable to create transaction"); + let frame = txn.get(frame_key.as_bytes(), false).await; + if let Ok(Some(data)) = frame { + return Some(data.to_vec().into()); + } + None + } + + async fn find_frame(&self, page_no: u64) -> Option { + let namespace = "default"; + let page_key = format!("{}/p/{}", namespace, page_no); + + let db = foundationdb::Database::default().unwrap(); + let txn = db.create_trx().expect("unable to create transaction"); + + let result = txn.get(&page_key.as_bytes(), false).await; + if let Err(e) = result { + error!("get failed: {:?}", e); + return None; + } + if let Ok(None) = result { + error!("page not found"); + return None; + } + let frame_no: u64 = unpack(&result.unwrap().unwrap()).expect("failed to decode u64"); + // let frame_no: u64 = unpack( + // &txn.get(&page_key.as_bytes(), true) + // .await + // .expect("get failed") + // .expect("frame not found"), + // ) + // .expect("failed to decode u64"); + Some(frame_no) + } + + async fn frame_page_no(&self, frame_no: u64) -> Option { + let namespace = "default"; + let frame_key = format!("{}/f/{}/p", namespace, frame_no); + + let db = foundationdb::Database::default().unwrap(); + let txn = db.create_trx().expect("unable to create transaction"); + let page_no: u64 = unpack( + &txn.get(&frame_key.as_bytes(), true) + .await + .expect("get failed") + .expect("frame not found"), + ) + .expect("failed to decode u64"); + + Some(page_no) + } + + async fn frames_in_wal(&self) -> u64 { + let namespace = "default"; + let db = foundationdb::Database::default().unwrap(); + let txn = db.create_trx().expect("unable to create transaction"); + self.get_max_frame_no(&txn, namespace).await + } + + async fn destroy(&mut self) {} +} diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index f711af7f14..acbf25c005 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -1,7 +1,8 @@ -mod memory_store; -mod redis_store; +mod fdb_store; + mod store; +use crate::fdb_store::FDBFrameStore; use crate::store::FrameStore; use anyhow::Result; use bytes::Bytes; @@ -14,10 +15,11 @@ use libsql_storage::rpc::{ }; use libsql_storage_server::version::Version; use redis::{Client, Commands, RedisResult}; -use redis_store::RedisFrameStore; +use std::env::args; use std::net::SocketAddr; use std::sync::atomic::AtomicU32; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; +use tokio::sync::Mutex; use tonic::{transport::Server, Request, Response, Status}; use tracing::{error, trace}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; @@ -39,14 +41,14 @@ struct FrameData { } struct Service { - store: Arc>, + store: Arc>, db_size: AtomicU32, } impl Service { - pub fn new(client: Client) -> Self { + pub fn new() -> Self { Self { - store: Arc::new(Mutex::new(RedisFrameStore::new(client))), + store: Arc::new(Mutex::new(FDBFrameStore::new())), db_size: AtomicU32::new(0), } } @@ -60,7 +62,7 @@ impl Storage for Service { ) -> Result, tonic::Status> { trace!("insert_frames()"); let mut num_frames = 0; - let mut store = self.store.lock().unwrap(); + let mut store = self.store.lock().await; trace!("insert_frames() got lock"); let frames = request .into_inner() @@ -81,7 +83,7 @@ impl Storage for Service { trace!( "inserted for page {} frame {}", frame.page_no, - store.insert_frame(frame.page_no, frame.data.into()) + store.insert_frame(frame.page_no, frame.data.into()).await ); num_frames += 1; self.db_size @@ -96,7 +98,7 @@ impl Storage for Service { ) -> Result, tonic::Status> { let page_no = request.into_inner().page_no; trace!("find_frame(page_no={})", page_no); - if let Some(frame_no) = self.store.lock().unwrap().find_frame(page_no) { + if let Some(frame_no) = self.store.lock().await.find_frame(page_no).await { Ok(Response::new(FindFrameResp { frame_no: Some(frame_no), })) @@ -112,7 +114,7 @@ impl Storage for Service { ) -> Result, tonic::Status> { let frame_no = request.into_inner().frame_no; trace!("read_frame(frame_no={})", frame_no); - if let Some(data) = self.store.lock().unwrap().read_frame(frame_no) { + if let Some(data) = self.store.lock().await.read_frame(frame_no).await { Ok(Response::new(ReadFrameResp { frame: Some(data.clone().into()), })) @@ -127,7 +129,7 @@ impl Storage for Service { request: tonic::Request, ) -> Result, tonic::Status> { trace!("destroy()"); - self.store.lock().unwrap().destroy(); + self.store.lock().await.destroy(); Ok(Response::new(DestroyResp {})) } @@ -144,7 +146,7 @@ impl Storage for Service { request: Request, ) -> std::result::Result, Status> { Ok(Response::new(FramesInWalResp { - count: self.store.lock().unwrap().frames_in_wal() as u32, + count: self.store.lock().await.frames_in_wal().await as u32, })) } @@ -153,7 +155,7 @@ impl Storage for Service { request: Request, ) -> std::result::Result, Status> { let frame_no = request.into_inner().frame_no; - if let Some(page_no) = self.store.lock().unwrap().frame_page_no(frame_no) { + if let Some(page_no) = self.store.lock().await.frame_page_no(frame_no).await { Ok(Response::new(FramePageNumResp { page_no })) } else { error!("frame_page_num() failed for frame_no={}", frame_no); @@ -172,11 +174,7 @@ async fn main() -> Result<()> { .expect("setting default subscriber failed"); let args = Cli::parse(); - // export REDIS_ADDR=http://libsql-storage-server.internal:5002 - let redis_addr = std::env::var("REDIS_ADDR").unwrap_or("redis://127.0.0.1/".to_string()); - let client = Client::open(redis_addr).unwrap(); - let service = Service::new(client); - + let service = Service::new(); println!("Starting libSQL storage server on {}", args.listen_addr); trace!( "(trace) Starting libSQL storage server on {}", diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index a10e474e26..c57d3eef27 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -1,6 +1,6 @@ use crate::store::FrameStore; -use redis::Client; -use redis::Commands; +use bytes::Bytes; +use redis::{Client, Commands, RedisResult}; use tracing::error; pub struct RedisFrameStore { diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index 5f1ed85b8e..c1472cb9d9 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -1,8 +1,8 @@ pub trait FrameStore { - fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64; - fn read_frame(&self, frame_no: u64) -> Option; - fn find_frame(&self, page_no: u64) -> Option; - fn frame_page_no(&self, frame_no: u64) -> Option; - fn frames_in_wal(&self) -> u64; - fn destroy(&mut self); + async fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64; + async fn read_frame(&self, frame_no: u64) -> Option; + async fn find_frame(&self, page_no: u64) -> Option; + async fn frame_page_no(&self, frame_no: u64) -> Option; + async fn frames_in_wal(&self) -> u64; + async fn destroy(&mut self); } From fb82f9699ffd7ce8519f5fc7cedfc8aeb7e3ce95 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 18 May 2024 21:33:28 +0530 Subject: [PATCH 34/60] Move redis methods to async --- libsql-storage-server/src/main.rs | 21 +++++++++++++++++---- libsql-storage-server/src/redis_store.rs | 12 ++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index acbf25c005..ac588aca0a 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -1,8 +1,10 @@ mod fdb_store; +mod redis_store; mod store; use crate::fdb_store::FDBFrameStore; +use crate::redis_store::RedisFrameStore; use crate::store::FrameStore; use anyhow::Result; use bytes::Bytes; @@ -41,14 +43,21 @@ struct FrameData { } struct Service { - store: Arc>, + // store: Arc>, + store: Arc>, db_size: AtomicU32, } impl Service { - pub fn new() -> Self { + // pub fn new() -> Self { + // Self { + // store: Arc::new(Mutex::new(FDBFrameStore::new())), + // db_size: AtomicU32::new(0), + // } + // } + pub fn new(client: Client) -> Self { Self { - store: Arc::new(Mutex::new(FDBFrameStore::new())), + store: Arc::new(Mutex::new(RedisFrameStore::new(client))), db_size: AtomicU32::new(0), } } @@ -174,7 +183,11 @@ async fn main() -> Result<()> { .expect("setting default subscriber failed"); let args = Cli::parse(); - let service = Service::new(); + // export REDIS_ADDR=http://libsql-storage-server.internal:5002 + let redis_addr = std::env::var("REDIS_ADDR").unwrap_or("redis://127.0.0.1/".to_string()); + let client = Client::open(redis_addr).unwrap(); + let service = Service::new(client); + // let service = Service::new(); println!("Starting libSQL storage server on {}", args.listen_addr); trace!( "(trace) Starting libSQL storage server on {}", diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index c57d3eef27..ee85678ca6 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -14,7 +14,7 @@ impl RedisFrameStore { } impl FrameStore for RedisFrameStore { - fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { + async fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { let namespace = "default"; let max_frame_key = format!("{}/max_frame_no", namespace); @@ -46,7 +46,7 @@ impl FrameStore for RedisFrameStore { max_frame_no } - fn read_frame(&self, frame_no: u64) -> Option { + async fn read_frame(&self, frame_no: u64) -> Option { let namespace = "default"; let frame_key = format!("f/{}/{}", namespace, frame_no); let mut con = self.client.get_connection().unwrap(); @@ -65,7 +65,7 @@ impl FrameStore for RedisFrameStore { } } - fn find_frame(&self, page_no: u64) -> Option { + async fn find_frame(&self, page_no: u64) -> Option { let page_key = format!("p/{}/{}", "default", page_no); let mut con = self.client.get_connection().unwrap(); let frame_no = con.get::(page_key.clone()); @@ -80,7 +80,7 @@ impl FrameStore for RedisFrameStore { } } - fn frame_page_no(&self, frame_no: u64) -> Option { + async fn frame_page_no(&self, frame_no: u64) -> Option { let namespace = "default"; let frame_key = format!("f/{}/{}", namespace, frame_no); let mut con = self.client.get_connection().unwrap(); @@ -99,7 +99,7 @@ impl FrameStore for RedisFrameStore { } } - fn frames_in_wal(&self) -> u64 { + async fn frames_in_wal(&self) -> u64 { let namespace = "default"; let max_frame_key = format!("{}/max_frame_no", namespace); let mut con = self.client.get_connection().unwrap(); @@ -112,7 +112,7 @@ impl FrameStore for RedisFrameStore { }) } - fn destroy(&mut self) { + async fn destroy(&mut self) { // remove all the keys in redis let mut con = self.client.get_connection().unwrap(); // send a FLUSHALL request From 7fd53798585fbfe947781376b94f2d607d31a63a Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 18 May 2024 21:33:42 +0530 Subject: [PATCH 35/60] fix bug while inserting mx_frame --- libsql-storage-server/src/fdb_store.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index 2f505b3330..4719d90962 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -40,7 +40,7 @@ impl FrameStore for FDBFrameStore { let max_frame_key = format!("{}/max_frame_no", namespace); let db = foundationdb::Database::default().unwrap(); let txn = db.create_trx().expect("unable to create transaction"); - let frame_no = self.get_max_frame_no(&txn, namespace).await; + let frame_no = self.get_max_frame_no(&txn, namespace).await + 1; let frame_key = format!("{}/f/{}/f", namespace, frame_no); let frame_page_key = format!("{}/f/{}/p", namespace, frame_no); @@ -49,9 +49,9 @@ impl FrameStore for FDBFrameStore { txn.set(&frame_key.as_bytes(), &frame); txn.set(&frame_page_key.as_bytes(), &pack(&page_no)); txn.set(&page_key.as_bytes(), &pack(&frame_no)); - txn.set(&max_frame_key.as_bytes(), &pack(&(frame_no + 1))); + txn.set(&max_frame_key.as_bytes(), &pack(&(frame_no))); txn.commit().await.expect("commit failed"); - frame_no + 1 + frame_no } async fn read_frame(&self, frame_no: u64) -> Option { @@ -84,13 +84,6 @@ impl FrameStore for FDBFrameStore { return None; } let frame_no: u64 = unpack(&result.unwrap().unwrap()).expect("failed to decode u64"); - // let frame_no: u64 = unpack( - // &txn.get(&page_key.as_bytes(), true) - // .await - // .expect("get failed") - // .expect("frame not found"), - // ) - // .expect("failed to decode u64"); Some(frame_no) } From 24e019c79676b9cf90110cb84ec0a78c5b1e9707 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Tue, 21 May 2024 19:41:22 +0530 Subject: [PATCH 36/60] Add a bulk insert method --- libsql-storage-server/src/fdb_store.rs | 51 ++++++++++++++++++++---- libsql-storage-server/src/main.rs | 22 +++++----- libsql-storage-server/src/redis_store.rs | 9 +++++ libsql-storage-server/src/store.rs | 3 ++ 4 files changed, 67 insertions(+), 18 deletions(-) diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index 4719d90962..3dab9474bd 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -1,4 +1,5 @@ use crate::store::FrameStore; +use crate::FrameData; use bytes::Bytes; use foundationdb::api::NetworkAutoStop; use foundationdb::tuple::pack; @@ -32,6 +33,17 @@ impl FDBFrameStore { tracing::info!("max_frame_no={}", frame_no); frame_no } + + async fn insert_with_tx(&mut self, txn: &Transaction, frame_no: u64, frame: FrameData) { + let namespace = "default"; + let frame_key = format!("{}/f/{}/f", namespace, frame_no); + let frame_page_key = format!("{}/f/{}/p", namespace, frame_no); + let page_key = format!("{}/p/{}", namespace, frame.page_no); + + txn.set(&frame_key.as_bytes(), &frame.data); + txn.set(&frame_page_key.as_bytes(), &pack(&frame.page_no)); + txn.set(&page_key.as_bytes(), &pack(&frame_no)); + } } impl FrameStore for FDBFrameStore { @@ -41,14 +53,39 @@ impl FrameStore for FDBFrameStore { let db = foundationdb::Database::default().unwrap(); let txn = db.create_trx().expect("unable to create transaction"); let frame_no = self.get_max_frame_no(&txn, namespace).await + 1; + self.insert_with_tx( + &txn, + frame_no, + FrameData { + page_no, + data: frame, + }, + ) + .await; + txn.set(&max_frame_key.as_bytes(), &pack(&(frame_no))); + txn.commit().await.expect("commit failed"); + frame_no + } - let frame_key = format!("{}/f/{}/f", namespace, frame_no); - let frame_page_key = format!("{}/f/{}/p", namespace, frame_no); - let page_key = format!("{}/p/{}", namespace, page_no); - - txn.set(&frame_key.as_bytes(), &frame); - txn.set(&frame_page_key.as_bytes(), &pack(&page_no)); - txn.set(&page_key.as_bytes(), &pack(&frame_no)); + async fn insert_frames(&mut self, frames: Vec) -> u64 { + let namespace = "default"; + let max_frame_key = format!("{}/max_frame_no", namespace); + let db = foundationdb::Database::default().unwrap(); + let txn = db.create_trx().expect("unable to create transaction"); + let mut frame_no = self.get_max_frame_no(&txn, namespace).await; + + for f in frames { + frame_no += 1; + self.insert_with_tx( + &txn, + frame_no, + FrameData { + page_no: f.page_no, + data: f.data, + }, + ) + .await; + } txn.set(&max_frame_key.as_bytes(), &pack(&(frame_no))); txn.commit().await.expect("commit failed"); frame_no diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index ac588aca0a..6d3fa3c44e 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -44,23 +44,23 @@ struct FrameData { struct Service { // store: Arc>, - store: Arc>, + store: Arc>, db_size: AtomicU32, } impl Service { - // pub fn new() -> Self { - // Self { - // store: Arc::new(Mutex::new(FDBFrameStore::new())), - // db_size: AtomicU32::new(0), - // } - // } - pub fn new(client: Client) -> Self { + pub fn new() -> Self { Self { - store: Arc::new(Mutex::new(RedisFrameStore::new(client))), + store: Arc::new(Mutex::new(FDBFrameStore::new())), db_size: AtomicU32::new(0), } } + // pub fn new(client: Client) -> Self { + // Self { + // store: Arc::new(Mutex::new(RedisFrameStore::new(client))), + // db_size: AtomicU32::new(0), + // } + // } } #[tonic::async_trait] @@ -186,8 +186,8 @@ async fn main() -> Result<()> { // export REDIS_ADDR=http://libsql-storage-server.internal:5002 let redis_addr = std::env::var("REDIS_ADDR").unwrap_or("redis://127.0.0.1/".to_string()); let client = Client::open(redis_addr).unwrap(); - let service = Service::new(client); - // let service = Service::new(); + // let service = Service::new(client); + let service = Service::new(); println!("Starting libSQL storage server on {}", args.listen_addr); trace!( "(trace) Starting libSQL storage server on {}", diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index ee85678ca6..5d615aa705 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -1,4 +1,5 @@ use crate::store::FrameStore; +use crate::FrameData; use bytes::Bytes; use redis::{Client, Commands, RedisResult}; use tracing::error; @@ -46,6 +47,14 @@ impl FrameStore for RedisFrameStore { max_frame_no } + async fn insert_frames(&mut self, frames: Vec) -> u64 { + let mut max_frame_no = 0; + for f in frames { + max_frame_no = self.insert_frame(f.page_no, f.data).await; + } + max_frame_no + } + async fn read_frame(&self, frame_no: u64) -> Option { let namespace = "default"; let frame_key = format!("f/{}/{}", namespace, frame_no); diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index c1472cb9d9..b5abd0c4d6 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -1,5 +1,8 @@ +use crate::FrameData; + pub trait FrameStore { async fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64; + async fn insert_frames(&mut self, frames: Vec) -> u64; async fn read_frame(&self, frame_no: u64) -> Option; async fn find_frame(&self, page_no: u64) -> Option; async fn frame_page_no(&self, frame_no: u64) -> Option; From c2834e0bce249f41a46edf0d8debd1366a86fcf2 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Tue, 21 May 2024 20:19:57 +0530 Subject: [PATCH 37/60] update proto to propagate ns --- libsql-storage/proto/storage.proto | 20 ++++++++++------ libsql-storage/src/generated/storage.rs | 32 +++++++++++++++++++------ libsql-storage/src/lib.rs | 19 +++++++++++---- 3 files changed, 53 insertions(+), 18 deletions(-) diff --git a/libsql-storage/proto/storage.proto b/libsql-storage/proto/storage.proto index ab6ce6edf5..aecb973def 100644 --- a/libsql-storage/proto/storage.proto +++ b/libsql-storage/proto/storage.proto @@ -8,7 +8,8 @@ message Frame { } message InsertFramesReq { - repeated Frame frames = 1; + string namespace = 1; + repeated Frame frames = 2; } message InsertFramesResp { @@ -16,7 +17,8 @@ message InsertFramesResp { } message FindFrameReq { - uint64 page_no = 1; + string namespace = 1; + uint64 page_no = 2; } message FindFrameResp { @@ -24,7 +26,8 @@ message FindFrameResp { } message ReadFrameReq { - uint64 frame_no = 1; + string namespace = 1; + uint64 frame_no = 2; } message ReadFrameResp { @@ -32,7 +35,7 @@ message ReadFrameResp { } message DbSizeReq { - + string namespace = 1; } message DbSizeResp { @@ -40,7 +43,7 @@ message DbSizeResp { } message FramesInWALReq { - + string namespace = 1; } message FramesInWALResp { @@ -48,14 +51,17 @@ message FramesInWALResp { } message FramePageNumReq { - uint64 frame_no = 1; + string namespace = 1; + uint64 frame_no = 2; } message FramePageNumResp { uint64 page_no = 1; } -message DestroyReq {} +message DestroyReq { + string namespace = 1; +} message DestroyResp {} diff --git a/libsql-storage/src/generated/storage.rs b/libsql-storage/src/generated/storage.rs index 2578e40f1e..5c6a68c953 100644 --- a/libsql-storage/src/generated/storage.rs +++ b/libsql-storage/src/generated/storage.rs @@ -1,3 +1,4 @@ +// This file is @generated by prost-build. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Frame { @@ -9,7 +10,9 @@ pub struct Frame { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct InsertFramesReq { - #[prost(message, repeated, tag = "1")] + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] pub frames: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] @@ -21,7 +24,9 @@ pub struct InsertFramesResp { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct FindFrameReq { - #[prost(uint64, tag = "1")] + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] pub page_no: u64, } #[allow(clippy::derive_partial_eq_without_eq)] @@ -33,7 +38,9 @@ pub struct FindFrameResp { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ReadFrameReq { - #[prost(uint64, tag = "1")] + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] pub frame_no: u64, } #[allow(clippy::derive_partial_eq_without_eq)] @@ -44,7 +51,10 @@ pub struct ReadFrameResp { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct DbSizeReq {} +pub struct DbSizeReq { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct DbSizeResp { @@ -53,7 +63,10 @@ pub struct DbSizeResp { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct FramesInWalReq {} +pub struct FramesInWalReq { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct FramesInWalResp { @@ -63,7 +76,9 @@ pub struct FramesInWalResp { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct FramePageNumReq { - #[prost(uint64, tag = "1")] + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] pub frame_no: u64, } #[allow(clippy::derive_partial_eq_without_eq)] @@ -74,7 +89,10 @@ pub struct FramePageNumResp { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct DestroyReq {} +pub struct DestroyReq { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct DestroyResp {} diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 9f37a5657c..4d735a46d3 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -95,6 +95,7 @@ pub struct DurableWal { runtime: Option, rt: tokio::runtime::Handle, write_cache: BTreeMap, + namespace: String, } impl DurableWal { @@ -123,8 +124,10 @@ impl DurableWal { // .unwrap(); let mut client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); - - let req = rpc::DbSizeReq {}; + let namespace = "default".to_string(); + let req = rpc::DbSizeReq { + namespace: namespace.clone(), + }; let resp = client.db_size(req); let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); let db_size = resp.into_inner().size as u32; @@ -140,6 +143,7 @@ impl DurableWal { runtime, rt, write_cache: BTreeMap::new(), + namespace, } } @@ -150,6 +154,7 @@ impl DurableWal { trace!("DurableWal::find_frame_by_page_no(page_no: {:?})", page_no); // TODO: send max frame number in the request let req = rpc::FindFrameReq { + namespace: self.namespace.clone(), page_no: page_no.get() as u64, }; let mut binding = self.client.lock(); @@ -164,7 +169,9 @@ impl DurableWal { } fn frames_count(&self) -> u32 { - let req = rpc::FramesInWalReq {}; + let req = rpc::FramesInWalReq { + namespace: self.namespace.clone(), + }; let mut binding = self.client.lock(); let resp = binding.frames_in_wal(req); let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); @@ -225,7 +232,10 @@ impl Wal for DurableWal { return Ok(()); } let frame_no = frame_no.get() as u64; - let req = rpc::ReadFrameReq { frame_no }; + let req = rpc::ReadFrameReq { + namespace: self.namespace.clone(), + frame_no, + }; let mut binding = self.client.lock(); let resp = binding.read_frame(req); let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); @@ -311,6 +321,7 @@ impl Wal for DurableWal { } let req = rpc::InsertFramesReq { + namespace: self.namespace.clone(), frames: self.write_cache.values().cloned().collect(), }; self.write_cache.clear(); From 1d5f84de14722fa77d5f2db6c83a30c282ab6ee0 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Tue, 21 May 2024 22:04:20 +0530 Subject: [PATCH 38/60] propagate namespace from server --- libsql-storage-server/src/fdb_store.rs | 38 +++++++-------- libsql-storage-server/src/main.rs | 60 +++++++++++++++++------- libsql-storage-server/src/redis_store.rs | 22 ++++----- libsql-storage-server/src/store.rs | 14 +++--- 4 files changed, 78 insertions(+), 56 deletions(-) diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index 3dab9474bd..a4bee08884 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -18,7 +18,6 @@ impl FDBFrameStore { } async fn get_max_frame_no(&self, txn: &Transaction, namespace: &str) -> u64 { - let namespace = "default"; let max_frame_key = format!("{}/max_frame_no", namespace); let result = txn.get(&max_frame_key.as_bytes(), false).await; if let Err(e) = result { @@ -30,30 +29,35 @@ impl FDBFrameStore { return 0; } let frame_no: u64 = unpack(&result.unwrap().unwrap()).expect("failed to decode u64"); - tracing::info!("max_frame_no={}", frame_no); + tracing::info!("max_frame_no ({}) = {}", max_frame_key, frame_no); frame_no } - async fn insert_with_tx(&mut self, txn: &Transaction, frame_no: u64, frame: FrameData) { - let namespace = "default"; - let frame_key = format!("{}/f/{}/f", namespace, frame_no); + async fn insert_with_tx( + &mut self, + namespace: &str, + txn: &Transaction, + frame_no: u64, + frame: FrameData, + ) { + let frame_data_key = format!("{}/f/{}/f", namespace, frame_no); let frame_page_key = format!("{}/f/{}/p", namespace, frame_no); let page_key = format!("{}/p/{}", namespace, frame.page_no); - txn.set(&frame_key.as_bytes(), &frame.data); + txn.set(&frame_data_key.as_bytes(), &frame.data); txn.set(&frame_page_key.as_bytes(), &pack(&frame.page_no)); txn.set(&page_key.as_bytes(), &pack(&frame_no)); } } impl FrameStore for FDBFrameStore { - async fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { - let namespace = "default"; + async fn insert_frame(&mut self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64 { let max_frame_key = format!("{}/max_frame_no", namespace); let db = foundationdb::Database::default().unwrap(); let txn = db.create_trx().expect("unable to create transaction"); let frame_no = self.get_max_frame_no(&txn, namespace).await + 1; self.insert_with_tx( + namespace, &txn, frame_no, FrameData { @@ -67,8 +71,7 @@ impl FrameStore for FDBFrameStore { frame_no } - async fn insert_frames(&mut self, frames: Vec) -> u64 { - let namespace = "default"; + async fn insert_frames(&mut self, namespace: &str, frames: Vec) -> u64 { let max_frame_key = format!("{}/max_frame_no", namespace); let db = foundationdb::Database::default().unwrap(); let txn = db.create_trx().expect("unable to create transaction"); @@ -77,6 +80,7 @@ impl FrameStore for FDBFrameStore { for f in frames { frame_no += 1; self.insert_with_tx( + namespace, &txn, frame_no, FrameData { @@ -91,8 +95,7 @@ impl FrameStore for FDBFrameStore { frame_no } - async fn read_frame(&self, frame_no: u64) -> Option { - let namespace = "default"; + async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option { let frame_key = format!("{}/f/{}/f", namespace, frame_no); let db = foundationdb::Database::default().unwrap(); @@ -104,8 +107,7 @@ impl FrameStore for FDBFrameStore { None } - async fn find_frame(&self, page_no: u64) -> Option { - let namespace = "default"; + async fn find_frame(&self, namespace: &str, page_no: u64) -> Option { let page_key = format!("{}/p/{}", namespace, page_no); let db = foundationdb::Database::default().unwrap(); @@ -124,8 +126,7 @@ impl FrameStore for FDBFrameStore { Some(frame_no) } - async fn frame_page_no(&self, frame_no: u64) -> Option { - let namespace = "default"; + async fn frame_page_no(&self, namespace: &str, frame_no: u64) -> Option { let frame_key = format!("{}/f/{}/p", namespace, frame_no); let db = foundationdb::Database::default().unwrap(); @@ -141,12 +142,11 @@ impl FrameStore for FDBFrameStore { Some(page_no) } - async fn frames_in_wal(&self) -> u64 { - let namespace = "default"; + async fn frames_in_wal(&self, namespace: &str) -> u64 { let db = foundationdb::Database::default().unwrap(); let txn = db.create_trx().expect("unable to create transaction"); self.get_max_frame_no(&txn, namespace).await } - async fn destroy(&mut self) {} + async fn destroy(&mut self, namespace: &str) {} } diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 6d3fa3c44e..0f6ef5ec83 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -73,14 +73,12 @@ impl Storage for Service { let mut num_frames = 0; let mut store = self.store.lock().await; trace!("insert_frames() got lock"); - let frames = request - .into_inner() - .frames - .into_iter() - .map(|frame| FrameData { - page_no: frame.page_no, - data: frame.data.into(), - }); + let request = request.into_inner(); + let namespace = request.namespace; + let frames = request.frames.into_iter().map(|frame| FrameData { + page_no: frame.page_no, + data: frame.data.into(), + }); let all_data: Vec = frames .clone() .map(|f| f.data.clone().to_vec()) @@ -92,7 +90,9 @@ impl Storage for Service { trace!( "inserted for page {} frame {}", frame.page_no, - store.insert_frame(frame.page_no, frame.data.into()).await + store + .insert_frame(&namespace, frame.page_no, frame.data.into()) + .await ); num_frames += 1; self.db_size @@ -105,9 +105,17 @@ impl Storage for Service { &self, request: tonic::Request, ) -> Result, tonic::Status> { - let page_no = request.into_inner().page_no; + let request = request.into_inner(); + let page_no = request.page_no; + let namespace = request.namespace; trace!("find_frame(page_no={})", page_no); - if let Some(frame_no) = self.store.lock().await.find_frame(page_no).await { + if let Some(frame_no) = self + .store + .lock() + .await + .find_frame(&namespace, page_no) + .await + { Ok(Response::new(FindFrameResp { frame_no: Some(frame_no), })) @@ -121,9 +129,17 @@ impl Storage for Service { &self, request: tonic::Request, ) -> Result, tonic::Status> { - let frame_no = request.into_inner().frame_no; + let request = request.into_inner(); + let frame_no = request.frame_no; + let namespace = request.namespace; trace!("read_frame(frame_no={})", frame_no); - if let Some(data) = self.store.lock().await.read_frame(frame_no).await { + if let Some(data) = self + .store + .lock() + .await + .read_frame(&namespace, frame_no) + .await + { Ok(Response::new(ReadFrameResp { frame: Some(data.clone().into()), })) @@ -138,7 +154,8 @@ impl Storage for Service { request: tonic::Request, ) -> Result, tonic::Status> { trace!("destroy()"); - self.store.lock().await.destroy(); + let namespace = request.into_inner().namespace; + self.store.lock().await.destroy(&namespace); Ok(Response::new(DestroyResp {})) } @@ -154,8 +171,9 @@ impl Storage for Service { &self, request: Request, ) -> std::result::Result, Status> { + let namespace = request.into_inner().namespace; Ok(Response::new(FramesInWalResp { - count: self.store.lock().await.frames_in_wal().await as u32, + count: self.store.lock().await.frames_in_wal(&namespace).await as u32, })) } @@ -163,8 +181,16 @@ impl Storage for Service { &self, request: Request, ) -> std::result::Result, Status> { - let frame_no = request.into_inner().frame_no; - if let Some(page_no) = self.store.lock().await.frame_page_no(frame_no).await { + let request = request.into_inner(); + let frame_no = request.frame_no; + let namespace = request.namespace; + if let Some(page_no) = self + .store + .lock() + .await + .frame_page_no(&namespace, frame_no) + .await + { Ok(Response::new(FramePageNumResp { page_no })) } else { error!("frame_page_num() failed for frame_no={}", frame_no); diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index 5d615aa705..4200aa8f79 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -15,8 +15,7 @@ impl RedisFrameStore { } impl FrameStore for RedisFrameStore { - async fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { - let namespace = "default"; + async fn insert_frame(&mut self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64 { let max_frame_key = format!("{}/max_frame_no", namespace); let mut con = self.client.get_connection().unwrap(); @@ -47,16 +46,15 @@ impl FrameStore for RedisFrameStore { max_frame_no } - async fn insert_frames(&mut self, frames: Vec) -> u64 { + async fn insert_frames(&mut self, namespace: &str, frames: Vec) -> u64 { let mut max_frame_no = 0; for f in frames { - max_frame_no = self.insert_frame(f.page_no, f.data).await; + max_frame_no = self.insert_frame(namespace, f.page_no, f.data).await; } max_frame_no } - async fn read_frame(&self, frame_no: u64) -> Option { - let namespace = "default"; + async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option { let frame_key = format!("f/{}/{}", namespace, frame_no); let mut con = self.client.get_connection().unwrap(); let result = con.hget::>(frame_key.clone(), "f"); @@ -74,8 +72,8 @@ impl FrameStore for RedisFrameStore { } } - async fn find_frame(&self, page_no: u64) -> Option { - let page_key = format!("p/{}/{}", "default", page_no); + async fn find_frame(&self, namespace: &str, page_no: u64) -> Option { + let page_key = format!("p/{}/{}", namespace, page_no); let mut con = self.client.get_connection().unwrap(); let frame_no = con.get::(page_key.clone()); match frame_no { @@ -89,8 +87,7 @@ impl FrameStore for RedisFrameStore { } } - async fn frame_page_no(&self, frame_no: u64) -> Option { - let namespace = "default"; + async fn frame_page_no(&self, namespace: &str, frame_no: u64) -> Option { let frame_key = format!("f/{}/{}", namespace, frame_no); let mut con = self.client.get_connection().unwrap(); let result = con.hget::(frame_key.clone(), "p"); @@ -108,8 +105,7 @@ impl FrameStore for RedisFrameStore { } } - async fn frames_in_wal(&self) -> u64 { - let namespace = "default"; + async fn frames_in_wal(&self, namespace: &str) -> u64 { let max_frame_key = format!("{}/max_frame_no", namespace); let mut con = self.client.get_connection().unwrap(); let result = con.get::(max_frame_key.clone()); @@ -121,7 +117,7 @@ impl FrameStore for RedisFrameStore { }) } - async fn destroy(&mut self) { + async fn destroy(&mut self, namespace: &str) { // remove all the keys in redis let mut con = self.client.get_connection().unwrap(); // send a FLUSHALL request diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index b5abd0c4d6..1ca0be32f4 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -1,11 +1,11 @@ use crate::FrameData; pub trait FrameStore { - async fn insert_frame(&mut self, page_no: u64, frame: bytes::Bytes) -> u64; - async fn insert_frames(&mut self, frames: Vec) -> u64; - async fn read_frame(&self, frame_no: u64) -> Option; - async fn find_frame(&self, page_no: u64) -> Option; - async fn frame_page_no(&self, frame_no: u64) -> Option; - async fn frames_in_wal(&self) -> u64; - async fn destroy(&mut self); + async fn insert_frame(&mut self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64; + async fn insert_frames(&mut self, namespace: &str, frames: Vec) -> u64; + async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option; + async fn find_frame(&self, namespace: &str, page_no: u64) -> Option; + async fn frame_page_no(&self, namespace: &str, frame_no: u64) -> Option; + async fn frames_in_wal(&self, namespace: &str) -> u64; + async fn destroy(&mut self, namespace: &str); } From 44ee1f841c3a4d671585ebb5a9ac1b6b1f9100c6 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Tue, 21 May 2024 22:36:15 +0530 Subject: [PATCH 39/60] Propagate max frame no --- libsql-storage/proto/storage.proto | 2 ++ libsql-storage/src/generated/storage.rs | 4 ++++ libsql-storage/src/lib.rs | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/libsql-storage/proto/storage.proto b/libsql-storage/proto/storage.proto index aecb973def..79bda7c935 100644 --- a/libsql-storage/proto/storage.proto +++ b/libsql-storage/proto/storage.proto @@ -10,6 +10,7 @@ message Frame { message InsertFramesReq { string namespace = 1; repeated Frame frames = 2; + uint64 max_frame_no = 3; } message InsertFramesResp { @@ -19,6 +20,7 @@ message InsertFramesResp { message FindFrameReq { string namespace = 1; uint64 page_no = 2; + uint64 max_frame_no = 3; } message FindFrameResp { diff --git a/libsql-storage/src/generated/storage.rs b/libsql-storage/src/generated/storage.rs index 5c6a68c953..4cb00837ae 100644 --- a/libsql-storage/src/generated/storage.rs +++ b/libsql-storage/src/generated/storage.rs @@ -14,6 +14,8 @@ pub struct InsertFramesReq { pub namespace: ::prost::alloc::string::String, #[prost(message, repeated, tag = "2")] pub frames: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "3")] + pub max_frame_no: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -28,6 +30,8 @@ pub struct FindFrameReq { pub namespace: ::prost::alloc::string::String, #[prost(uint64, tag = "2")] pub page_no: u64, + #[prost(uint64, tag = "3")] + pub max_frame_no: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 4d735a46d3..03ed927d9a 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -152,10 +152,10 @@ impl DurableWal { page_no: std::num::NonZeroU32, ) -> Result> { trace!("DurableWal::find_frame_by_page_no(page_no: {:?})", page_no); - // TODO: send max frame number in the request let req = rpc::FindFrameReq { namespace: self.namespace.clone(), page_no: page_no.get() as u64, + max_frame_no: 0, }; let mut binding = self.client.lock(); let resp = binding.find_frame(req); @@ -323,6 +323,7 @@ impl Wal for DurableWal { let req = rpc::InsertFramesReq { namespace: self.namespace.clone(), frames: self.write_cache.values().cloned().collect(), + max_frame_no: 0, }; self.write_cache.clear(); let mut binding = self.client.lock(); From e5d174891f5eb55fb75d6f9433524155cc0fef00 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 30 May 2024 13:32:56 +0530 Subject: [PATCH 40/60] Add storage server proto definition --- Cargo.lock | 28 +- Cargo.toml | 1 + libsql-storage/Cargo.toml | 15 + libsql-storage/proto/storage.proto | 78 +++ libsql-storage/src/generated/storage.rs | 838 ++++++++++++++++++++++++ libsql-storage/src/lib.rs | 4 + libsql-storage/tests/bootstrap.rs | 33 + 7 files changed, 996 insertions(+), 1 deletion(-) create mode 100644 libsql-storage/Cargo.toml create mode 100644 libsql-storage/proto/storage.proto create mode 100644 libsql-storage/src/generated/storage.rs create mode 100644 libsql-storage/src/lib.rs create mode 100644 libsql-storage/tests/bootstrap.rs diff --git a/Cargo.lock b/Cargo.lock index 87d8db6c8b..803252c0fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3449,6 +3449,16 @@ dependencies = [ "uncased", ] +[[package]] +name = "libsql-storage" +version = "0.0.1" +dependencies = [ + "prost", + "prost-build", + "tonic 0.10.2", + "tonic-build 0.10.2", +] + [[package]] name = "libsql-sys" version = "0.5.0" @@ -3535,7 +3545,7 @@ dependencies = [ "tokio-stream", "tokio-util", "tonic 0.11.0", - "tonic-build", + "tonic-build 0.11.0", "tracing", "uuid", "zerocopy", @@ -5717,7 +5727,10 @@ dependencies = [ "percent-encoding", "pin-project", "prost", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", "tokio", + "tokio-rustls 0.24.1", "tokio-stream", "tower", "tower-layer", @@ -5757,6 +5770,19 @@ dependencies = [ "webpki-roots 0.26.1", ] +[[package]] +name = "tonic-build" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn 2.0.66", +] + [[package]] name = "tonic-build" version = "0.11.0" diff --git a/Cargo.toml b/Cargo.toml index 7b59faae80..e2eaf2ffff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "xtask", "libsql-hrana", "libsql-wal", + "libsql-storage", ] exclude = [ diff --git a/libsql-storage/Cargo.toml b/libsql-storage/Cargo.toml new file mode 100644 index 0000000000..94346eef60 --- /dev/null +++ b/libsql-storage/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "libsql-storage" +version = "0.0.1" +edition = "2021" +description = "libSQL WAL" +repository = "https://github.com/tursodatabase/libsql" +license = "MIT" + +[dependencies] +prost = "0.12" +tonic = { version = "0.10", features = ["tls"] } + +[dev-dependencies] +prost-build = "0.12" +tonic-build = "0.10" diff --git a/libsql-storage/proto/storage.proto b/libsql-storage/proto/storage.proto new file mode 100644 index 0000000000..a46c4445cc --- /dev/null +++ b/libsql-storage/proto/storage.proto @@ -0,0 +1,78 @@ +syntax = "proto3"; + +package storage; + +message Frame { + uint64 page_no = 1; + bytes data = 2; +} + +message InsertFramesRequest { + string namespace = 1; + repeated Frame frames = 2; + uint64 max_frame_no = 3; +} + +message InsertFramesResponse { + uint32 num_frames = 1; +} + +message FindFrameRequest { + string namespace = 1; + uint64 page_no = 2; + uint64 max_frame_no = 3; +} + +message FindFrameResponse { + optional uint64 frame_no = 1; +} + +message ReadFrameRequest { + string namespace = 1; + uint64 frame_no = 2; +} + +message ReadFrameResponse { + optional bytes frame = 1; +} + +message DbSizeRequest { + string namespace = 1; +} + +message DbSizeResponse { + uint64 size = 1; +} + +message FramesInWALRequest { + string namespace = 1; +} + +message FramesInWALResponse { + uint64 count = 1; +} + +message FramePageNumRequest { + string namespace = 1; + uint64 frame_no = 2; +} + +message FramePageNumResponse { + uint64 page_no = 1; +} + +message DestroyRequest { + string namespace = 1; +} + +message DestroyResponse {} + +service Storage { + rpc InsertFrames(InsertFramesRequest) returns (InsertFramesResponse) {} + rpc FindFrame(FindFrameRequest) returns (FindFrameResponse) {} + rpc ReadFrame(ReadFrameRequest) returns (ReadFrameResponse) {} + rpc DbSize(DbSizeRequest) returns (DbSizeResponse) {} + rpc FramesInWAL(FramesInWALRequest) returns (FramesInWALResponse) {} + rpc FramePageNum(FramePageNumRequest) returns (FramePageNumResponse) {} + rpc Destroy(DestroyRequest) returns (DestroyResponse) {} +} diff --git a/libsql-storage/src/generated/storage.rs b/libsql-storage/src/generated/storage.rs new file mode 100644 index 0000000000..d5be7c451a --- /dev/null +++ b/libsql-storage/src/generated/storage.rs @@ -0,0 +1,838 @@ +// This file is @generated by prost-build. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Frame { + #[prost(uint64, tag = "1")] + pub page_no: u64, + #[prost(bytes = "vec", tag = "2")] + pub data: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct InsertFramesRequest { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub frames: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "3")] + pub max_frame_no: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct InsertFramesResponse { + #[prost(uint32, tag = "1")] + pub num_frames: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FindFrameRequest { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub page_no: u64, + #[prost(uint64, tag = "3")] + pub max_frame_no: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FindFrameResponse { + #[prost(uint64, optional, tag = "1")] + pub frame_no: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadFrameRequest { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub frame_no: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadFrameResponse { + #[prost(bytes = "vec", optional, tag = "1")] + pub frame: ::core::option::Option<::prost::alloc::vec::Vec>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DbSizeRequest { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DbSizeResponse { + #[prost(uint64, tag = "1")] + pub size: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FramesInWalRequest { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FramesInWalResponse { + #[prost(uint64, tag = "1")] + pub count: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FramePageNumRequest { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub frame_no: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FramePageNumResponse { + #[prost(uint64, tag = "1")] + pub page_no: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DestroyRequest { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DestroyResponse {} +/// Generated client implementations. +pub mod storage_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct StorageClient { + inner: tonic::client::Grpc, + } + impl StorageClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl StorageClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> StorageClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + Send + Sync, + { + StorageClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn insert_frames( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/storage.Storage/InsertFrames", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("storage.Storage", "InsertFrames")); + self.inner.unary(req, path, codec).await + } + pub async fn find_frame( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/storage.Storage/FindFrame", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("storage.Storage", "FindFrame")); + self.inner.unary(req, path, codec).await + } + pub async fn read_frame( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/storage.Storage/ReadFrame", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("storage.Storage", "ReadFrame")); + self.inner.unary(req, path, codec).await + } + pub async fn db_size( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/storage.Storage/DbSize"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("storage.Storage", "DbSize")); + self.inner.unary(req, path, codec).await + } + pub async fn frames_in_wal( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/storage.Storage/FramesInWAL", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("storage.Storage", "FramesInWAL")); + self.inner.unary(req, path, codec).await + } + pub async fn frame_page_num( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/storage.Storage/FramePageNum", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("storage.Storage", "FramePageNum")); + self.inner.unary(req, path, codec).await + } + pub async fn destroy( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/storage.Storage/Destroy"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("storage.Storage", "Destroy")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod storage_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with StorageServer. + #[async_trait] + pub trait Storage: Send + Sync + 'static { + async fn insert_frames( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + async fn find_frame( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + async fn read_frame( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + async fn db_size( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn frames_in_wal( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + async fn frame_page_num( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + async fn destroy( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct StorageServer { + inner: _Inner, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + struct _Inner(Arc); + impl StorageServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + let inner = _Inner(inner); + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for StorageServer + where + T: Storage, + B: Body + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/storage.Storage/InsertFrames" => { + #[allow(non_camel_case_types)] + struct InsertFramesSvc(pub Arc); + impl< + T: Storage, + > tonic::server::UnaryService + for InsertFramesSvc { + type Response = super::InsertFramesResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::insert_frames(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = InsertFramesSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/storage.Storage/FindFrame" => { + #[allow(non_camel_case_types)] + struct FindFrameSvc(pub Arc); + impl tonic::server::UnaryService + for FindFrameSvc { + type Response = super::FindFrameResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::find_frame(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = FindFrameSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/storage.Storage/ReadFrame" => { + #[allow(non_camel_case_types)] + struct ReadFrameSvc(pub Arc); + impl tonic::server::UnaryService + for ReadFrameSvc { + type Response = super::ReadFrameResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::read_frame(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = ReadFrameSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/storage.Storage/DbSize" => { + #[allow(non_camel_case_types)] + struct DbSizeSvc(pub Arc); + impl tonic::server::UnaryService + for DbSizeSvc { + type Response = super::DbSizeResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::db_size(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = DbSizeSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/storage.Storage/FramesInWAL" => { + #[allow(non_camel_case_types)] + struct FramesInWALSvc(pub Arc); + impl< + T: Storage, + > tonic::server::UnaryService + for FramesInWALSvc { + type Response = super::FramesInWalResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::frames_in_wal(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = FramesInWALSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/storage.Storage/FramePageNum" => { + #[allow(non_camel_case_types)] + struct FramePageNumSvc(pub Arc); + impl< + T: Storage, + > tonic::server::UnaryService + for FramePageNumSvc { + type Response = super::FramePageNumResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::frame_page_num(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = FramePageNumSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/storage.Storage/Destroy" => { + #[allow(non_camel_case_types)] + struct DestroySvc(pub Arc); + impl tonic::server::UnaryService + for DestroySvc { + type Response = super::DestroyResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::destroy(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = DestroySvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + Ok( + http::Response::builder() + .status(200) + .header("grpc-status", "12") + .header("content-type", "application/grpc") + .body(empty_body()) + .unwrap(), + ) + }) + } + } + } + } + impl Clone for StorageServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::server::NamedService for StorageServer { + const NAME: &'static str = "storage.Storage"; + } +} diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs new file mode 100644 index 0000000000..c01585a1cb --- /dev/null +++ b/libsql-storage/src/lib.rs @@ -0,0 +1,4 @@ +pub mod rpc { + #![allow(clippy::all)] + include!("generated/storage.rs"); +} diff --git a/libsql-storage/tests/bootstrap.rs b/libsql-storage/tests/bootstrap.rs new file mode 100644 index 0000000000..e259f0e3b3 --- /dev/null +++ b/libsql-storage/tests/bootstrap.rs @@ -0,0 +1,33 @@ +use std::{path::PathBuf, process::Command}; + +#[test] +fn bootstrap() { + let iface_files = &["proto/storage.proto"]; + let dirs = &["storage"]; + + let out_dir = PathBuf::from(std::env!("CARGO_MANIFEST_DIR")) + .join("src") + .join("generated"); + + let mut config = prost_build::Config::new(); + config.bytes([".wal_log"]); + + tonic_build::configure() + .build_client(true) + .build_server(true) + .build_transport(true) + .out_dir(&out_dir) + .type_attribute(".proxy", "#[derive(serde::Serialize, serde::Deserialize)]") + .compile_with_config(config, iface_files, dirs) + .unwrap(); + + let status = Command::new("git") + .arg("diff") + .arg("--exit-code") + .arg("--") + .arg(&out_dir) + .status() + .unwrap(); + + assert!(status.success(), "You should commit the protobuf files"); +} From 467675a197b8fa46d8db075fe167a39a26739c6f Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 30 May 2024 17:25:43 +0530 Subject: [PATCH 41/60] update clients --- libsql-storage-server/src/main.rs | 84 ++++++++++++++++--------------- libsql-storage/src/lib.rs | 20 ++++---- 2 files changed, 53 insertions(+), 51 deletions(-) diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 0f6ef5ec83..f1f9273a78 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -1,31 +1,33 @@ -mod fdb_store; - -mod redis_store; -mod store; +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::AtomicU32; -use crate::fdb_store::FDBFrameStore; -use crate::redis_store::RedisFrameStore; -use crate::store::FrameStore; use anyhow::Result; use bytes::Bytes; use clap::Parser; -use libsql_storage::rpc::storage_server::{Storage, StorageServer}; use libsql_storage::rpc::{ - DbSizeReq, DbSizeResp, DestroyReq, DestroyResp, FindFrameReq, FindFrameResp, FramePageNumReq, - FramePageNumResp, FramesInWalReq, FramesInWalResp, InsertFramesReq, InsertFramesResp, - ReadFrameReq, ReadFrameResp, + DbSizeRequest, DbSizeResponse, DestroyRequest, DestroyResponse, FindFrameRequest, + FindFrameResponse, FramePageNumRequest, FramePageNumResponse, FramesInWalRequest, + FramesInWalResponse, InsertFramesRequest, InsertFramesResponse, ReadFrameRequest, + ReadFrameResponse, }; +use libsql_storage::rpc::storage_server::{Storage, StorageServer}; use libsql_storage_server::version::Version; use redis::{Client, Commands, RedisResult}; -use std::env::args; -use std::net::SocketAddr; -use std::sync::atomic::AtomicU32; -use std::sync::Arc; use tokio::sync::Mutex; -use tonic::{transport::Server, Request, Response, Status}; +use tonic::{Request, Response, Status, transport::Server}; use tracing::{error, trace}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; +use crate::fdb_store::FDBFrameStore; +use crate::redis_store::RedisFrameStore; +use crate::store::FrameStore; + +mod fdb_store; + +mod redis_store; +mod store; + /// libSQL storage server #[derive(Debug, Parser)] #[command(name = "libsql-storage-server")] @@ -67,8 +69,8 @@ impl Service { impl Storage for Service { async fn insert_frames( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { + request: tonic::Request, + ) -> Result, tonic::Status> { trace!("insert_frames()"); let mut num_frames = 0; let mut store = self.store.lock().await; @@ -98,13 +100,13 @@ impl Storage for Service { self.db_size .fetch_add(1, std::sync::atomic::Ordering::SeqCst); } - Ok(Response::new(InsertFramesResp { num_frames })) + Ok(Response::new(InsertFramesResponse { num_frames })) } async fn find_frame( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { + request: tonic::Request, + ) -> Result, tonic::Status> { let request = request.into_inner(); let page_no = request.page_no; let namespace = request.namespace; @@ -116,19 +118,19 @@ impl Storage for Service { .find_frame(&namespace, page_no) .await { - Ok(Response::new(FindFrameResp { + Ok(Response::new(FindFrameResponse { frame_no: Some(frame_no), })) } else { error!("find_frame() failed for page_no={}", page_no); - Ok(Response::new(FindFrameResp { frame_no: None })) + Ok(Response::new(FindFrameResponse { frame_no: None })) } } async fn read_frame( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { + request: tonic::Request, + ) -> Result, tonic::Status> { let request = request.into_inner(); let frame_no = request.frame_no; let namespace = request.namespace; @@ -140,47 +142,47 @@ impl Storage for Service { .read_frame(&namespace, frame_no) .await { - Ok(Response::new(ReadFrameResp { + Ok(Response::new(ReadFrameResponse { frame: Some(data.clone().into()), })) } else { error!("read_frame() failed for frame_no={}", frame_no); - Ok(Response::new(ReadFrameResp { frame: None })) + Ok(Response::new(ReadFrameResponse { frame: None })) } } async fn destroy( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { + request: tonic::Request, + ) -> Result, tonic::Status> { trace!("destroy()"); let namespace = request.into_inner().namespace; self.store.lock().await.destroy(&namespace); - Ok(Response::new(DestroyResp {})) + Ok(Response::new(DestroyResponse {})) } async fn db_size( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { + request: tonic::Request, + ) -> Result, tonic::Status> { let size = self.db_size.load(std::sync::atomic::Ordering::SeqCst) as u64; - Ok(Response::new(DbSizeResp { size })) + Ok(Response::new(DbSizeResponse { size })) } async fn frames_in_wal( &self, - request: Request, - ) -> std::result::Result, Status> { + request: Request, + ) -> std::result::Result, Status> { let namespace = request.into_inner().namespace; - Ok(Response::new(FramesInWalResp { - count: self.store.lock().await.frames_in_wal(&namespace).await as u32, + Ok(Response::new(FramesInWalResponse { + count: self.store.lock().await.frames_in_wal(&namespace).await, })) } async fn frame_page_num( &self, - request: Request, - ) -> std::result::Result, Status> { + request: Request, + ) -> std::result::Result, Status> { let request = request.into_inner(); let frame_no = request.frame_no; let namespace = request.namespace; @@ -191,10 +193,10 @@ impl Storage for Service { .frame_page_no(&namespace, frame_no) .await { - Ok(Response::new(FramePageNumResp { page_no })) + Ok(Response::new(FramePageNumResponse { page_no })) } else { error!("frame_page_num() failed for frame_no={}", frame_no); - Ok(Response::new(FramePageNumResp { page_no: 0 })) + Ok(Response::new(FramePageNumResponse { page_no: 0 })) } } } diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 7ce38f3fa1..afacc72b65 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -67,8 +67,8 @@ impl WalManager for DurableWalManager { // let client = StorageClient::connect(address); // let rt = tokio::runtime::Runtime::new().unwrap(); // let mut client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); - // let req = rpc::DestroyReq {}; - // let resp = client.destroy(req); + // let Request = rpc::DestroyRequest {}; + // let resp = client.destroy(Request); // let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); Ok(()) } @@ -126,7 +126,7 @@ impl DurableWal { let mut client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); let namespace = "default".to_string(); - let req = rpc::DbSizeReq { + let req = rpc::DbSizeRequest { namespace: namespace.clone(), }; let resp = client.db_size(req); @@ -153,7 +153,7 @@ impl DurableWal { page_no: std::num::NonZeroU32, ) -> Result> { trace!("DurableWal::find_frame_by_page_no(page_no: {:?})", page_no); - let req = rpc::FindFrameReq { + let req = rpc::FindFrameRequest { namespace: self.namespace.clone(), page_no: page_no.get() as u64, max_frame_no: 0, @@ -169,8 +169,8 @@ impl DurableWal { Ok(frame_no) } - fn frames_count(&self) -> u32 { - let req = rpc::FramesInWalReq { + fn frames_count(&self) -> u64 { + let req = rpc::FramesInWalRequest { namespace: self.namespace.clone(), }; let mut binding = self.client.lock(); @@ -233,7 +233,7 @@ impl Wal for DurableWal { return Ok(()); } let frame_no = frame_no.get() as u64; - let req = rpc::ReadFrameReq { + let req = rpc::ReadFrameRequest { namespace: self.namespace.clone(), frame_no, }; @@ -249,7 +249,7 @@ impl Wal for DurableWal { fn db_size(&self) -> u32 { trace!("DurableWal::db_size() => {}", self.db_size); - self.frames_count() + self.frames_count().try_into().unwrap() } fn begin_write_txn(&mut self) -> Result<()> { @@ -321,7 +321,7 @@ impl Wal for DurableWal { return Ok(0); } - let req = rpc::InsertFramesReq { + let req = rpc::InsertFramesRequest { namespace: self.namespace.clone(), frames: self.write_cache.values().cloned().collect(), max_frame_no: 0, @@ -373,7 +373,7 @@ impl Wal for DurableWal { } fn frames_in_wal(&self) -> u32 { - // let req = rpc::FramesInWalReq {}; + // let req = rpc::FramesInWalRequest {}; // let mut binding = self.client.lock(); // let resp = binding.frames_in_wal(req); // let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); From cb76a4df68183c5aed2355d6a7f29a541cf76d07 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Fri, 31 May 2024 14:50:32 +0530 Subject: [PATCH 42/60] refactor --- libsql-server/src/connection/mod.rs | 2 +- libsql-storage/src/lib.rs | 229 ++++++++++++++-------------- 2 files changed, 112 insertions(+), 119 deletions(-) diff --git a/libsql-server/src/connection/mod.rs b/libsql-server/src/connection/mod.rs index e554130a80..6d786dae3e 100644 --- a/libsql-server/src/connection/mod.rs +++ b/libsql-server/src/connection/mod.rs @@ -31,7 +31,7 @@ pub mod program; pub mod write_proxy; #[cfg(not(test))] -const TXN_TIMEOUT: Duration = Duration::from_secs(5); +const TXN_TIMEOUT: Duration = Duration::from_secs(500); #[cfg(test)] const TXN_TIMEOUT: Duration = Duration::from_millis(100); diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index afacc72b65..47ad5a25f8 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -1,22 +1,27 @@ use std::collections::BTreeMap; -use std::mem::size_of; use std::sync::{Arc, Mutex}; -use tonic::transport::Channel; - -use libsql_sys::ffi::SQLITE_BUSY; +use libsql_sys::ffi::{SQLITE_ABORT, SQLITE_BUSY}; use libsql_sys::rusqlite; use libsql_sys::wal::{Result, Vfs, Wal, WalManager}; use rpc::storage_client::StorageClient; use sieve_cache::SieveCache; -use tracing::trace; -use uuid::uuid; +use tonic::transport::Channel; +use tracing::{error, trace}; pub mod rpc { #![allow(clippy::all)] include!("generated/storage.rs"); } +// TODO / status: +// - there are no read txn locks nor upgrades +// - no lock stealing +// - write set is kept in mem +// - no savepoints, yet +// - no multi tenancy, uses `default` namespace +// - txn can read new frames after it started (since there are no read locks) + #[derive(Clone)] pub struct DurableWalManager { lock_manager: Arc>, @@ -38,44 +43,38 @@ impl WalManager for DurableWalManager { fn open( &self, - vfs: &mut Vfs, - file: &mut libsql_sys::wal::Sqlite3File, - no_shm_mode: std::ffi::c_int, - max_log_size: i64, + _vfs: &mut Vfs, + _file: &mut libsql_sys::wal::Sqlite3File, + _no_shm_mode: std::ffi::c_int, + _max_log_size: i64, db_path: &std::ffi::CStr, ) -> Result { let db_path = db_path.to_str().unwrap(); trace!("DurableWalManager::open(db_path: {})", db_path); - Ok(DurableWal::new(self.lock_manager.clone())) + // TODO: use the actual namespace uuid from the connection + let namespace = "default".to_string(); + Ok(DurableWal::new(namespace, self.lock_manager.clone())) } fn close( &self, wal: &mut Self::Wal, - db: &mut libsql_sys::wal::Sqlite3Db, - sync_flags: std::ffi::c_int, - scratch: Option<&mut [u8]>, + _db: &mut libsql_sys::wal::Sqlite3Db, + _sync_flags: std::ffi::c_int, + _scratch: Option<&mut [u8]>, ) -> Result<()> { trace!("DurableWalManager::close()"); + wal.end_read_txn(); Ok(()) } - fn destroy_log(&self, vfs: &mut Vfs, db_path: &std::ffi::CStr) -> Result<()> { + fn destroy_log(&self, _vfs: &mut Vfs, _db_path: &std::ffi::CStr) -> Result<()> { trace!("DurableWalManager::destroy_log()"); - // let address = std::env::var("LIBSQL_STORAGE_SERVER_ADDR") - // .unwrap_or("http://127.0.0.1:5002".to_string()); - // let client = StorageClient::connect(address); - // let rt = tokio::runtime::Runtime::new().unwrap(); - // let mut client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); - // let Request = rpc::DestroyRequest {}; - // let resp = client.destroy(Request); - // let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); Ok(()) } - fn log_exists(&self, vfs: &mut Vfs, db_path: &std::ffi::CStr) -> Result { + fn log_exists(&self, _vfs: &mut Vfs, _db_path: &std::ffi::CStr) -> Result { trace!("DurableWalManager::log_exists()"); - // TODO: implement Ok(true) } @@ -88,20 +87,19 @@ impl WalManager for DurableWalManager { } pub struct DurableWal { + namespace: String, + conn_id: String, client: parking_lot::Mutex>, - page_frames: SieveCache>, - db_size: u32, - name: String, + frames_cache: SieveCache>, + write_cache: BTreeMap, lock_manager: Arc>, - runtime: Option, + _runtime: Option, rt: tokio::runtime::Handle, - write_cache: BTreeMap, - namespace: String, } impl DurableWal { - fn new(lock_manager: Arc>) -> Self { - let (runtime, rt) = match tokio::runtime::Handle::try_current() { + fn new(namespace: String, lock_manager: Arc>) -> Self { + let (_runtime, rt) = match tokio::runtime::Handle::try_current() { Ok(h) => (None, h), Err(_) => { let rt = tokio::runtime::Runtime::new().unwrap(); @@ -113,45 +111,26 @@ impl DurableWal { // export LIBSQL_STORAGE_SERVER_ADDR=http://libsql-storage-server.internal:5002 let address = std::env::var("LIBSQL_STORAGE_SERVER_ADDR") .unwrap_or("http://127.0.0.1:5002".to_string()); - trace!("DurableWal::new() address = {}", address); - - // let channel = Channel::from_static(address).connect_lazy().unwrap(); - // let channel = Channel::builder("127.0.0.1:3000".parse().unwrap()).connect_lazy(); - let client = StorageClient::connect(address); - // let client = client.max_encoding_message_size(100); - // let client = StorageClient::new(channel) - // .max_encoding_message_size(100) - // .unwrap(); - - let mut client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); - let namespace = "default".to_string(); - let req = rpc::DbSizeRequest { - namespace: namespace.clone(), - }; - let resp = client.db_size(req); - let resp = tokio::task::block_in_place(|| rt.block_on(resp)).unwrap(); - let db_size = resp.into_inner().size as u32; - + let client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); let page_frames = SieveCache::new(1000).unwrap(); Self { + namespace, + conn_id: uuid::Uuid::new_v4().to_string(), client: parking_lot::Mutex::new(client), - page_frames, - db_size, - name: uuid::Uuid::new_v4().to_string(), + frames_cache: page_frames, + write_cache: BTreeMap::new(), lock_manager, - runtime, + _runtime, rt, - write_cache: BTreeMap::new(), - namespace, } } fn find_frame_by_page_no( &mut self, page_no: std::num::NonZeroU32, - ) -> Result> { + ) -> Result> { trace!("DurableWal::find_frame_by_page_no(page_no: {:?})", page_no); let req = rpc::FindFrameRequest { namespace: self.namespace.clone(), @@ -164,7 +143,7 @@ impl DurableWal { let frame_no = resp .into_inner() .frame_no - .map(|page_no| std::num::NonZeroU32::new(page_no as u32)) + .map(|no| std::num::NonZeroU64::new(no)) .flatten(); Ok(frame_no) } @@ -183,27 +162,38 @@ impl DurableWal { } impl Wal for DurableWal { - fn limit(&mut self, size: i64) { - // no op, we go bottomless baby! - } + fn limit(&mut self, _size: i64) {} fn begin_read_txn(&mut self) -> Result { trace!("DurableWal::begin_read_txn()"); - // TODO: give a read lock for this conn - // note down max frame number + // TODO: + // - create a read lock + // - save the current max_frame_no for this txn + // Ok(true) } fn end_read_txn(&mut self) { trace!("DurableWal::end_read_txn()"); // TODO: drop both read or write lock + let mut lock_manager = self.lock_manager.lock().unwrap(); + trace!( + "DurableWal::end_read_txn() id = {}, unlocked = {}", + self.conn_id, + lock_manager.unlock(self.namespace.clone(), self.conn_id.clone()) + ); } + // find_frame checks if the given page_no exists in the storage server. If so, it returns the + // same `page_no` back. The WAL interface expects the value to be u32 but the frames can exceed + // the limit and is set to u64. So, instead of returning the frame no, it returns the page no + // back and `read_frame` methods reads the frame by page_no fn find_frame( &mut self, page_no: std::num::NonZeroU32, ) -> Result> { trace!("DurableWal::find_frame(page_no: {:?})", page_no); + // TODO: find_frame should account for `max_frame_no` of this txn let frame_no = self.find_frame_by_page_no(page_no).unwrap(); if frame_no.is_none() { return Ok(None); @@ -211,7 +201,6 @@ impl Wal for DurableWal { return Ok(Some(page_no)); } - // read_frame reads the page, not the frame fn read_frame(&mut self, page_no: std::num::NonZeroU32, buffer: &mut [u8]) -> Result<()> { trace!("DurableWal::read_frame(page_no: {:?})", page_no); if let Some(frame) = self.write_cache.get(&(u32::from(page_no))) { @@ -222,9 +211,10 @@ impl Wal for DurableWal { buffer.copy_from_slice(&frame.data); return Ok(()); } + // TODO: this call is unnecessary since `read_frame` is always called after `find_frame` let frame_no = self.find_frame_by_page_no(page_no).unwrap().unwrap(); // check if the frame exists in the local cache - if let Some(frame) = self.page_frames.get(&frame_no) { + if let Some(frame) = self.frames_cache.get(&frame_no) { trace!( "DurableWal::read_frame(page_no: {:?}) -- read cache hit", page_no @@ -232,63 +222,64 @@ impl Wal for DurableWal { buffer.copy_from_slice(&frame); return Ok(()); } - let frame_no = frame_no.get() as u64; let req = rpc::ReadFrameRequest { namespace: self.namespace.clone(), - frame_no, + frame_no: frame_no.get(), }; let mut binding = self.client.lock(); let resp = binding.read_frame(req); let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); let frame = resp.into_inner().frame.unwrap(); buffer.copy_from_slice(&frame); - self.page_frames - .insert(std::num::NonZeroU32::new(frame_no as u32).unwrap(), frame); + self.frames_cache + .insert(std::num::NonZeroU64::new(frame_no.get()).unwrap(), frame); Ok(()) } fn db_size(&self) -> u32 { - trace!("DurableWal::db_size() => {}", self.db_size); - self.frames_count().try_into().unwrap() + let size = self.frames_count().try_into().unwrap(); + trace!("DurableWal::db_size() => {}", size); + size } fn begin_write_txn(&mut self) -> Result<()> { - // todo: check if the connection holds a read lock - // then try to acquire a write lock + // todo: check if the connection holds a read lock then try to acquire a write lock let mut lock_manager = self.lock_manager.lock().unwrap(); - if !lock_manager.lock("default".to_string(), self.name.clone()) { + if !lock_manager.lock(self.namespace.clone(), self.conn_id.clone()) { trace!( - "DurableWal::begin_write_txn() lock = false, id = {}", - self.name + "DurableWal::begin_write_txn() lock acquired = false, id = {}", + self.conn_id ); return Err(rusqlite::ffi::Error::new(SQLITE_BUSY)); }; trace!( - "DurableWal::begin_write_txn() lock = true, id = {}", - self.name + "DurableWal::begin_write_txn() lock acquired = true, id = {}", + self.conn_id ); Ok(()) } fn end_write_txn(&mut self) -> Result<()> { - // release only if lock is write lock let mut lock_manager = self.lock_manager.lock().unwrap(); trace!( "DurableWal::end_write_txn() id = {}, unlocked = {}", - self.name, - lock_manager.unlock("default".to_string(), self.name.clone()) + self.conn_id, + lock_manager.unlock(self.namespace.clone(), self.conn_id.clone()) ); Ok(()) } - fn undo(&mut self, handler: Option<&mut U>) -> Result<()> { - // TODO: no op + fn undo(&mut self, _handler: Option<&mut U>) -> Result<()> { + // TODO: implement undo Ok(()) } - fn savepoint(&mut self, rollback_data: &mut [u32]) {} + fn savepoint(&mut self, _rollback_data: &mut [u32]) { + // TODO: implement savepoint + } - fn savepoint_undo(&mut self, rollback_data: &mut [u32]) -> Result<()> { + fn savepoint_undo(&mut self, _rollback_data: &mut [u32]) -> Result<()> { + // TODO: implement savepoint_undo Ok(()) } @@ -300,11 +291,15 @@ impl Wal for DurableWal { is_commit: bool, sync_flags: std::ffi::c_int, ) -> Result { - // TODO: check if it has a write lock - // check if the size_after is > 0, if so then mark txn as committed - trace!("name = {}", self.name); + trace!("name = {}", self.conn_id); trace!("DurableWal::insert_frames(page_size: {}, size_after: {}, is_commit: {}, sync_flags: {})", page_size, size_after, is_commit, sync_flags); - // add data from frame_headers to writeCache + let mut lock_manager = self.lock_manager.lock().unwrap(); + if !lock_manager.is_lock_owner(self.namespace.clone(), self.conn_id.clone()) { + error!("DurableWal::insert_frames() was called without acquiring lock!",); + self.write_cache.clear(); + return Err(rusqlite::ffi::Error::new(SQLITE_ABORT)); + }; + // add the updated frames from frame_headers to writeCache for (page_no, frame) in page_headers.iter() { self.write_cache.insert( page_no, @@ -316,6 +311,7 @@ impl Wal for DurableWal { // todo: update size after } + // check if the size_after is > 0, if so then mark txn as committed if size_after <= 0 { // todo: update new size return Ok(0); @@ -329,27 +325,22 @@ impl Wal for DurableWal { self.write_cache.clear(); let mut binding = self.client.lock(); trace!("sending DurableWal::insert_frames() {:?}", req.frames.len()); - let mut c = binding - .clone() - .max_encoding_message_size(256 * 1024 * 1024) - .max_decoding_message_size(256 * 1024 * 1024); - let resp = c.insert_frames(req); + let resp = binding.insert_frames(req); let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); - self.db_size = size_after; Ok(resp.into_inner().num_frames as usize) } fn checkpoint( &mut self, - db: &mut libsql_sys::wal::Sqlite3Db, - mode: libsql_sys::wal::CheckpointMode, - busy_handler: Option<&mut dyn libsql_sys::wal::BusyHandler>, - sync_flags: u32, + _db: &mut libsql_sys::wal::Sqlite3Db, + _mode: libsql_sys::wal::CheckpointMode, + _busy_handler: Option<&mut dyn libsql_sys::wal::BusyHandler>, + _sync_flags: u32, // temporary scratch buffer - buf: &mut [u8], - checkpoint_cb: Option<&mut dyn libsql_sys::wal::CheckpointCallback>, - in_wal: Option<&mut i32>, - backfilled: Option<&mut i32>, + _buf: &mut [u8], + _checkpoint_cb: Option<&mut dyn libsql_sys::wal::CheckpointCallback>, + _in_wal: Option<&mut i32>, + _backfilled: Option<&mut i32>, ) -> Result<()> { // checkpoint is a no op Ok(()) @@ -365,7 +356,7 @@ impl Wal for DurableWal { true } - fn set_db(&mut self, db: &mut libsql_sys::wal::Sqlite3Db) {} + fn set_db(&mut self, _db: &mut libsql_sys::wal::Sqlite3Db) {} fn callback(&self) -> i32 { trace!("DurableWal::callback()"); @@ -373,13 +364,6 @@ impl Wal for DurableWal { } fn frames_in_wal(&self) -> u32 { - // let req = rpc::FramesInWalRequest {}; - // let mut binding = self.client.lock(); - // let resp = binding.frames_in_wal(req); - // let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); - // let count = resp.into_inner().count; - // trace!("DurableWal::frames_in_wal() = {}", count); - // count 0 } } @@ -395,20 +379,20 @@ impl LockManager { } } - pub fn lock(&mut self, namespace: String, wal_id: String) -> bool { + pub fn lock(&mut self, namespace: String, conn_id: String) -> bool { if let Some(lock) = self.locks.get(&namespace) { - if lock == &wal_id { + if lock == &conn_id { return true; } return false; } - self.locks.insert(namespace, wal_id); + self.locks.insert(namespace, conn_id); true } - pub fn unlock(&mut self, namespace: String, wal_id: String) -> bool { + pub fn unlock(&mut self, namespace: String, conn_id: String) -> bool { if let Some(lock) = self.locks.get(&namespace) { - if lock == &wal_id { + if lock == &conn_id { self.locks.remove(&namespace); return true; } @@ -416,4 +400,13 @@ impl LockManager { } true } + + pub fn is_lock_owner(&mut self, namespace: String, conn_id: String) -> bool { + if let Some(lock) = self.locks.get(&namespace) { + if lock == &conn_id { + return true; + } + } + return false; + } } From 913d87e3ec14c5f4f7836a2e3f51483234153aa8 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Fri, 31 May 2024 20:27:58 +0530 Subject: [PATCH 43/60] Refactor to have pluggable store --- Cargo.lock | 2 + libsql-storage-server/Cargo.toml | 4 +- libsql-storage-server/src/fdb_store.rs | 2 + libsql-storage-server/src/main.rs | 64 +++++++++++++---------- libsql-storage-server/src/memory_store.rs | 20 ++++--- libsql-storage-server/src/redis_store.rs | 2 + libsql-storage-server/src/store.rs | 4 +- 7 files changed, 62 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b321e4ed7e..257f34f003 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3557,12 +3557,14 @@ name = "libsql-storage-server" version = "0.23.0" dependencies = [ "anyhow", + "async-trait", "bytes", "clap 4.5.4", "foundationdb", "futures", "libsql-storage", "redis", + "serde", "tokio", "tonic 0.10.2", "tracing", diff --git a/libsql-storage-server/Cargo.toml b/libsql-storage-server/Cargo.toml index 7eca66e379..d6dd36afed 100644 --- a/libsql-storage-server/Cargo.toml +++ b/libsql-storage-server/Cargo.toml @@ -11,7 +11,7 @@ path = "src/main.rs" [dependencies] anyhow = "1.0.66" bytes = "1.5.0" -clap = { version = "4.0.23", features = [ "derive", "env", "string" ] } +clap = { version = "4.0.23", features = ["derive", "env", "string"] } foundationdb = { version = "0.9.0", features = ["embedded-fdb-include", "fdb-7_3"] } futures = "0.3.30" libsql-storage = { path = "../libsql-storage" } @@ -20,6 +20,8 @@ tokio = { version = "1.22.2", features = ["rt-multi-thread", "net", "io-std", "i tonic = { version = "0.10.0", features = ["tls"] } tracing = "0.1.37" tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } +async-trait = "0.1.80" +serde = "1.0.203" [dev-dependencies] diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index a4bee08884..86dff1aaed 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -1,5 +1,6 @@ use crate::store::FrameStore; use crate::FrameData; +use async_trait::async_trait; use bytes::Bytes; use foundationdb::api::NetworkAutoStop; use foundationdb::tuple::pack; @@ -50,6 +51,7 @@ impl FDBFrameStore { } } +#[async_trait] impl FrameStore for FDBFrameStore { async fn insert_frame(&mut self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64 { let max_frame_key = format!("{}/max_frame_no", namespace); diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index f1f9273a78..f494e5dce7 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -1,34 +1,42 @@ use std::net::SocketAddr; -use std::sync::Arc; use std::sync::atomic::AtomicU32; +use std::sync::Arc; use anyhow::Result; use bytes::Bytes; use clap::Parser; +use libsql_storage::rpc::storage_server::{Storage, StorageServer}; use libsql_storage::rpc::{ DbSizeRequest, DbSizeResponse, DestroyRequest, DestroyResponse, FindFrameRequest, FindFrameResponse, FramePageNumRequest, FramePageNumResponse, FramesInWalRequest, FramesInWalResponse, InsertFramesRequest, InsertFramesResponse, ReadFrameRequest, ReadFrameResponse, }; -use libsql_storage::rpc::storage_server::{Storage, StorageServer}; use libsql_storage_server::version::Version; use redis::{Client, Commands, RedisResult}; +use serde; use tokio::sync::Mutex; -use tonic::{Request, Response, Status, transport::Server}; +use tonic::{transport::Server, Request, Response, Status}; use tracing::{error, trace}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; use crate::fdb_store::FDBFrameStore; +use crate::memory_store::InMemFrameStore; use crate::redis_store::RedisFrameStore; use crate::store::FrameStore; mod fdb_store; - +mod memory_store; mod redis_store; mod store; -/// libSQL storage server +#[derive(clap::ValueEnum, Clone, Debug)] +enum StorageType { + InMemory, + Redis, + FoundationDB, +} + #[derive(Debug, Parser)] #[command(name = "libsql-storage-server")] #[command(about = "libSQL storage server", version = Version::default(), long_about = None)] @@ -36,33 +44,35 @@ struct Cli { /// The address and port the storage RPC protocol listens to. Example: `127.0.0.1:5002`. #[clap(long, env = "LIBSQL_STORAGE_LISTEN_ADDR", default_value = "[::]:5002")] listen_addr: SocketAddr, + + #[clap(value_enum, long, default_value = "in-memory")] + storage_type: StorageType, } #[derive(Default)] struct FrameData { page_no: u64, - data: bytes::Bytes, + data: Bytes, } struct Service { - // store: Arc>, - store: Arc>, + store: Arc>, db_size: AtomicU32, } impl Service { pub fn new() -> Self { Self { - store: Arc::new(Mutex::new(FDBFrameStore::new())), + store: Arc::new(Mutex::new(InMemFrameStore::new())), + db_size: AtomicU32::new(0), + } + } + pub fn with_store(store: Arc>) -> Self { + Self { + store, db_size: AtomicU32::new(0), } } - // pub fn new(client: Client) -> Self { - // Self { - // store: Arc::new(Mutex::new(RedisFrameStore::new(client))), - // db_size: AtomicU32::new(0), - // } - // } } #[tonic::async_trait] @@ -151,16 +161,6 @@ impl Storage for Service { } } - async fn destroy( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - trace!("destroy()"); - let namespace = request.into_inner().namespace; - self.store.lock().await.destroy(&namespace); - Ok(Response::new(DestroyResponse {})) - } - async fn db_size( &self, request: tonic::Request, @@ -199,6 +199,16 @@ impl Storage for Service { Ok(Response::new(FramePageNumResponse { page_no: 0 })) } } + + async fn destroy( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + trace!("destroy()"); + let namespace = request.into_inner().namespace; + self.store.lock().await.destroy(&namespace).await; + Ok(Response::new(DestroyResponse {})) + } } #[tokio::main] @@ -214,8 +224,8 @@ async fn main() -> Result<()> { // export REDIS_ADDR=http://libsql-storage-server.internal:5002 let redis_addr = std::env::var("REDIS_ADDR").unwrap_or("redis://127.0.0.1/".to_string()); let client = Client::open(redis_addr).unwrap(); - // let service = Service::new(client); - let service = Service::new(); + Arc::new(Mutex::new(RedisFrameStore::new(client))); + let service = Service::with_store(Arc::new(Mutex::new(FDBFrameStore::new()))); println!("Starting libSQL storage server on {}", args.listen_addr); trace!( "(trace) Starting libSQL storage server on {}", diff --git a/libsql-storage-server/src/memory_store.rs b/libsql-storage-server/src/memory_store.rs index cd1c2ac9fd..6f7396dce3 100644 --- a/libsql-storage-server/src/memory_store.rs +++ b/libsql-storage-server/src/memory_store.rs @@ -1,10 +1,11 @@ use crate::store::FrameStore; use crate::FrameData; +use async_trait::async_trait; use bytes::Bytes; use std::collections::BTreeMap; #[derive(Default)] -struct InMemFrameStore { +pub(crate) struct InMemFrameStore { // contains a frame data, key is the frame number frames: BTreeMap, // pages map contains the page number as a key and the list of frames for the page as a value @@ -18,9 +19,10 @@ impl InMemFrameStore { } } +#[async_trait] impl FrameStore for InMemFrameStore { // inserts a new frame for the page number and returns the new frame value - fn insert_frame(&mut self, page_no: u64, frame: Bytes) -> u64 { + async fn insert_frame(&mut self, namespace: &str, page_no: u64, frame: Bytes) -> u64 { let frame_no = self.max_frame_no + 1; self.max_frame_no = frame_no; self.frames.insert( @@ -37,27 +39,31 @@ impl FrameStore for InMemFrameStore { frame_no } - fn read_frame(&self, frame_no: u64) -> Option { + async fn insert_frames(&mut self, namespace: &str, frames: Vec) -> u64 { + todo!() + } + + async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option { self.frames.get(&frame_no).map(|frame| frame.data.clone()) } // given a page number, return the maximum frame for the page - fn find_frame(&self, page_no: u64) -> Option { + async fn find_frame(&self, namespace: &str, page_no: u64) -> Option { self.pages .get(&page_no) .map(|frames| *frames.last().unwrap()) } // given a frame num, return the page number - fn frame_page_no(&self, frame_no: u64) -> Option { + async fn frame_page_no(&self, namespace: &str, frame_no: u64) -> Option { self.frames.get(&frame_no).map(|frame| frame.page_no) } - fn frames_in_wal(&self) -> u64 { + async fn frames_in_wal(&self, namespace: &str) -> u64 { self.max_frame_no } - fn destroy(&mut self) { + async fn destroy(&mut self, namespace: &str) { self.frames.clear(); self.pages.clear(); self.max_frame_no = 0; diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index 4200aa8f79..9883df71ad 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -1,5 +1,6 @@ use crate::store::FrameStore; use crate::FrameData; +use async_trait::async_trait; use bytes::Bytes; use redis::{Client, Commands, RedisResult}; use tracing::error; @@ -14,6 +15,7 @@ impl RedisFrameStore { } } +#[async_trait] impl FrameStore for RedisFrameStore { async fn insert_frame(&mut self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64 { let max_frame_key = format!("{}/max_frame_no", namespace); diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index 1ca0be32f4..b1cf9367f9 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -1,6 +1,8 @@ use crate::FrameData; +use async_trait::async_trait; -pub trait FrameStore { +#[async_trait] +pub trait FrameStore: Send + Sync { async fn insert_frame(&mut self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64; async fn insert_frames(&mut self, namespace: &str, frames: Vec) -> u64; async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option; From b2def08c394a95181a94f205c96ef7a2e1160fc1 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Fri, 31 May 2024 20:40:03 +0530 Subject: [PATCH 44/60] refactor to service --- libsql-storage-server/src/fdb_store.rs | 2 +- libsql-storage-server/src/main.rs | 170 +-------------------- libsql-storage-server/src/memory_store.rs | 2 +- libsql-storage-server/src/redis_store.rs | 16 +- libsql-storage-server/src/service.rs | 171 ++++++++++++++++++++++ libsql-storage-server/src/store.rs | 8 +- 6 files changed, 192 insertions(+), 177 deletions(-) create mode 100644 libsql-storage-server/src/service.rs diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index 86dff1aaed..400e1877be 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -1,5 +1,5 @@ +use crate::store::FrameData; use crate::store::FrameStore; -use crate::FrameData; use async_trait::async_trait; use bytes::Bytes; use foundationdb::api::NetworkAutoStop; diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index f494e5dce7..3b1eb1e2ea 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -15,6 +15,7 @@ use libsql_storage::rpc::{ use libsql_storage_server::version::Version; use redis::{Client, Commands, RedisResult}; use serde; +use service::Service; use tokio::sync::Mutex; use tonic::{transport::Server, Request, Response, Status}; use tracing::{error, trace}; @@ -28,6 +29,7 @@ use crate::store::FrameStore; mod fdb_store; mod memory_store; mod redis_store; +mod service; mod store; #[derive(clap::ValueEnum, Clone, Debug)] @@ -49,168 +51,6 @@ struct Cli { storage_type: StorageType, } -#[derive(Default)] -struct FrameData { - page_no: u64, - data: Bytes, -} - -struct Service { - store: Arc>, - db_size: AtomicU32, -} - -impl Service { - pub fn new() -> Self { - Self { - store: Arc::new(Mutex::new(InMemFrameStore::new())), - db_size: AtomicU32::new(0), - } - } - pub fn with_store(store: Arc>) -> Self { - Self { - store, - db_size: AtomicU32::new(0), - } - } -} - -#[tonic::async_trait] -impl Storage for Service { - async fn insert_frames( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - trace!("insert_frames()"); - let mut num_frames = 0; - let mut store = self.store.lock().await; - trace!("insert_frames() got lock"); - let request = request.into_inner(); - let namespace = request.namespace; - let frames = request.frames.into_iter().map(|frame| FrameData { - page_no: frame.page_no, - data: frame.data.into(), - }); - let all_data: Vec = frames - .clone() - .map(|f| f.data.clone().to_vec()) - .flatten() - .collect(); - trace!("insert_frames() got frames (bytes): {:?}", all_data.len()); - trace!("insert_frames() got frames: {:?}", frames.len()); - for frame in frames { - trace!( - "inserted for page {} frame {}", - frame.page_no, - store - .insert_frame(&namespace, frame.page_no, frame.data.into()) - .await - ); - num_frames += 1; - self.db_size - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - } - Ok(Response::new(InsertFramesResponse { num_frames })) - } - - async fn find_frame( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let request = request.into_inner(); - let page_no = request.page_no; - let namespace = request.namespace; - trace!("find_frame(page_no={})", page_no); - if let Some(frame_no) = self - .store - .lock() - .await - .find_frame(&namespace, page_no) - .await - { - Ok(Response::new(FindFrameResponse { - frame_no: Some(frame_no), - })) - } else { - error!("find_frame() failed for page_no={}", page_no); - Ok(Response::new(FindFrameResponse { frame_no: None })) - } - } - - async fn read_frame( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let request = request.into_inner(); - let frame_no = request.frame_no; - let namespace = request.namespace; - trace!("read_frame(frame_no={})", frame_no); - if let Some(data) = self - .store - .lock() - .await - .read_frame(&namespace, frame_no) - .await - { - Ok(Response::new(ReadFrameResponse { - frame: Some(data.clone().into()), - })) - } else { - error!("read_frame() failed for frame_no={}", frame_no); - Ok(Response::new(ReadFrameResponse { frame: None })) - } - } - - async fn db_size( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let size = self.db_size.load(std::sync::atomic::Ordering::SeqCst) as u64; - Ok(Response::new(DbSizeResponse { size })) - } - - async fn frames_in_wal( - &self, - request: Request, - ) -> std::result::Result, Status> { - let namespace = request.into_inner().namespace; - Ok(Response::new(FramesInWalResponse { - count: self.store.lock().await.frames_in_wal(&namespace).await, - })) - } - - async fn frame_page_num( - &self, - request: Request, - ) -> std::result::Result, Status> { - let request = request.into_inner(); - let frame_no = request.frame_no; - let namespace = request.namespace; - if let Some(page_no) = self - .store - .lock() - .await - .frame_page_no(&namespace, frame_no) - .await - { - Ok(Response::new(FramePageNumResponse { page_no })) - } else { - error!("frame_page_num() failed for frame_no={}", frame_no); - Ok(Response::new(FramePageNumResponse { page_no: 0 })) - } - } - - async fn destroy( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - trace!("destroy()"); - let namespace = request.into_inner().namespace; - self.store.lock().await.destroy(&namespace).await; - Ok(Response::new(DestroyResponse {})) - } -} - #[tokio::main] async fn main() -> Result<()> { let filter = EnvFilter::try_from_default_env() @@ -224,9 +64,7 @@ async fn main() -> Result<()> { // export REDIS_ADDR=http://libsql-storage-server.internal:5002 let redis_addr = std::env::var("REDIS_ADDR").unwrap_or("redis://127.0.0.1/".to_string()); let client = Client::open(redis_addr).unwrap(); - Arc::new(Mutex::new(RedisFrameStore::new(client))); let service = Service::with_store(Arc::new(Mutex::new(FDBFrameStore::new()))); - println!("Starting libSQL storage server on {}", args.listen_addr); trace!( "(trace) Starting libSQL storage server on {}", args.listen_addr @@ -238,7 +76,3 @@ async fn main() -> Result<()> { Ok(()) } - -fn is_nil_response(e: &redis::RedisError) -> bool { - e.to_string().contains("response was nil") -} diff --git a/libsql-storage-server/src/memory_store.rs b/libsql-storage-server/src/memory_store.rs index 6f7396dce3..17ec0d17d8 100644 --- a/libsql-storage-server/src/memory_store.rs +++ b/libsql-storage-server/src/memory_store.rs @@ -1,5 +1,5 @@ +use crate::store::FrameData; use crate::store::FrameStore; -use crate::FrameData; use async_trait::async_trait; use bytes::Bytes; use std::collections::BTreeMap; diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index 9883df71ad..43c9fc67a5 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -1,5 +1,5 @@ +use crate::store::FrameData; use crate::store::FrameStore; -use crate::FrameData; use async_trait::async_trait; use bytes::Bytes; use redis::{Client, Commands, RedisResult}; @@ -26,7 +26,7 @@ impl FrameStore for RedisFrameStore { let (max_frame_no,): (u64,) = redis::transaction(&mut con, &[&max_frame_key], |con, pipe| { let result: RedisResult = con.get(max_frame_key.clone()); - if result.is_err() && !crate::is_nil_response(result.as_ref().err().unwrap()) { + if result.is_err() && !is_nil_response(result.as_ref().err().unwrap()) { return Err(result.err().unwrap()); } let max_frame_no = result.unwrap_or(0) + 1; @@ -63,7 +63,7 @@ impl FrameStore for RedisFrameStore { match result { Ok(frame) => Some(Bytes::from(frame)), Err(e) => { - if !crate::is_nil_response(&e) { + if !is_nil_response(&e) { error!( "read_frame() failed for frame_no={} with err={}", frame_no, e @@ -81,7 +81,7 @@ impl FrameStore for RedisFrameStore { match frame_no { Ok(frame_no) => Some(frame_no), Err(e) => { - if !crate::is_nil_response(&e) { + if !is_nil_response(&e) { error!("find_frame() failed for page_no={} with err={}", page_no, e); } None @@ -96,7 +96,7 @@ impl FrameStore for RedisFrameStore { match result { Ok(page_no) => Some(page_no), Err(e) => { - if !crate::is_nil_response(&e) { + if !is_nil_response(&e) { error!( "frame_page_no() failed for frame_no={} with err={}", frame_no, e @@ -112,7 +112,7 @@ impl FrameStore for RedisFrameStore { let mut con = self.client.get_connection().unwrap(); let result = con.get::(max_frame_key.clone()); result.unwrap_or_else(|e| { - if !crate::is_nil_response(&e) { + if !is_nil_response(&e) { error!("frames_in_wal() failed with err={}", e); } 0 @@ -126,3 +126,7 @@ impl FrameStore for RedisFrameStore { let _: () = redis::cmd("FLUSHALL").query(&mut con).unwrap(); } } + +fn is_nil_response(e: &redis::RedisError) -> bool { + e.to_string().contains("response was nil") +} diff --git a/libsql-storage-server/src/service.rs b/libsql-storage-server/src/service.rs new file mode 100644 index 0000000000..dc2da90ef9 --- /dev/null +++ b/libsql-storage-server/src/service.rs @@ -0,0 +1,171 @@ +use crate::memory_store::InMemFrameStore; +use crate::store::{FrameData, FrameStore}; +use bytes::Bytes; +use libsql_storage::rpc::storage_server::Storage; +use libsql_storage::rpc::{ + DbSizeRequest, DbSizeResponse, DestroyRequest, DestroyResponse, FindFrameRequest, + FindFrameResponse, FramePageNumRequest, FramePageNumResponse, FramesInWalRequest, + FramesInWalResponse, InsertFramesRequest, InsertFramesResponse, ReadFrameRequest, + ReadFrameResponse, +}; +use std::sync::atomic::AtomicU32; +use std::sync::Arc; +use tokio::sync::Mutex; +use tonic::{Request, Response, Status}; +use tracing::{error, trace}; + +pub struct Service { + store: Arc>, + db_size: AtomicU32, +} + +impl Service { + pub fn new() -> Self { + Self { + store: Arc::new(Mutex::new(InMemFrameStore::new())), + db_size: AtomicU32::new(0), + } + } + pub fn with_store(store: Arc>) -> Self { + Self { + store, + db_size: AtomicU32::new(0), + } + } +} + +#[tonic::async_trait] +impl Storage for Service { + async fn insert_frames( + &self, + request: tonic::Request, + ) -> anyhow::Result, tonic::Status> { + trace!("insert_frames()"); + let mut num_frames = 0; + let mut store = self.store.lock().await; + trace!("insert_frames() got lock"); + let request = request.into_inner(); + let namespace = request.namespace; + let frames = request.frames.into_iter().map(|frame| FrameData { + page_no: frame.page_no, + data: frame.data.into(), + }); + let all_data: Vec = frames + .clone() + .map(|f| f.data.clone().to_vec()) + .flatten() + .collect(); + trace!("insert_frames() got frames (bytes): {:?}", all_data.len()); + trace!("insert_frames() got frames: {:?}", frames.len()); + for frame in frames { + trace!( + "inserted for page {} frame {}", + frame.page_no, + store + .insert_frame(&namespace, frame.page_no, frame.data.into()) + .await + ); + num_frames += 1; + self.db_size + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + Ok(Response::new(InsertFramesResponse { num_frames })) + } + + async fn find_frame( + &self, + request: tonic::Request, + ) -> anyhow::Result, tonic::Status> { + let request = request.into_inner(); + let page_no = request.page_no; + let namespace = request.namespace; + trace!("find_frame(page_no={})", page_no); + if let Some(frame_no) = self + .store + .lock() + .await + .find_frame(&namespace, page_no) + .await + { + Ok(Response::new(FindFrameResponse { + frame_no: Some(frame_no), + })) + } else { + error!("find_frame() failed for page_no={}", page_no); + Ok(Response::new(FindFrameResponse { frame_no: None })) + } + } + + async fn read_frame( + &self, + request: tonic::Request, + ) -> anyhow::Result, tonic::Status> { + let request = request.into_inner(); + let frame_no = request.frame_no; + let namespace = request.namespace; + trace!("read_frame(frame_no={})", frame_no); + if let Some(data) = self + .store + .lock() + .await + .read_frame(&namespace, frame_no) + .await + { + Ok(Response::new(ReadFrameResponse { + frame: Some(data.clone().into()), + })) + } else { + error!("read_frame() failed for frame_no={}", frame_no); + Ok(Response::new(ReadFrameResponse { frame: None })) + } + } + + async fn db_size( + &self, + request: tonic::Request, + ) -> anyhow::Result, tonic::Status> { + let size = self.db_size.load(std::sync::atomic::Ordering::SeqCst) as u64; + Ok(Response::new(DbSizeResponse { size })) + } + + async fn frames_in_wal( + &self, + request: Request, + ) -> std::result::Result, Status> { + let namespace = request.into_inner().namespace; + Ok(Response::new(FramesInWalResponse { + count: self.store.lock().await.frames_in_wal(&namespace).await, + })) + } + + async fn frame_page_num( + &self, + request: Request, + ) -> std::result::Result, Status> { + let request = request.into_inner(); + let frame_no = request.frame_no; + let namespace = request.namespace; + if let Some(page_no) = self + .store + .lock() + .await + .frame_page_no(&namespace, frame_no) + .await + { + Ok(Response::new(FramePageNumResponse { page_no })) + } else { + error!("frame_page_num() failed for frame_no={}", frame_no); + Ok(Response::new(FramePageNumResponse { page_no: 0 })) + } + } + + async fn destroy( + &self, + request: tonic::Request, + ) -> anyhow::Result, tonic::Status> { + trace!("destroy()"); + let namespace = request.into_inner().namespace; + self.store.lock().await.destroy(&namespace).await; + Ok(Response::new(DestroyResponse {})) + } +} diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index b1cf9367f9..fcd38835b0 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -1,5 +1,5 @@ -use crate::FrameData; use async_trait::async_trait; +use bytes::Bytes; #[async_trait] pub trait FrameStore: Send + Sync { @@ -11,3 +11,9 @@ pub trait FrameStore: Send + Sync { async fn frames_in_wal(&self, namespace: &str) -> u64; async fn destroy(&mut self, namespace: &str); } + +#[derive(Default)] +pub struct FrameData { + pub(crate) page_no: u64, + pub(crate) data: Bytes, +} From df14f1233ea778f80fc88281347ec626e893912a Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Fri, 31 May 2024 21:28:31 +0530 Subject: [PATCH 45/60] cargo fix --- libsql-storage-server/src/fdb_store.rs | 1 - libsql-storage-server/src/main.rs | 49 +++++++++++----------- libsql-storage-server/src/service.rs | 56 ++++++++++++-------------- 3 files changed, 49 insertions(+), 57 deletions(-) diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index 400e1877be..c2784b449d 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -1,7 +1,6 @@ use crate::store::FrameData; use crate::store::FrameStore; use async_trait::async_trait; -use bytes::Bytes; use foundationdb::api::NetworkAutoStop; use foundationdb::tuple::pack; use foundationdb::tuple::unpack; diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 3b1eb1e2ea..7eea0cf592 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -1,31 +1,19 @@ use std::net::SocketAddr; -use std::sync::atomic::AtomicU32; use std::sync::Arc; +use crate::fdb_store::FDBFrameStore; +use crate::redis_store::RedisFrameStore; use anyhow::Result; -use bytes::Bytes; use clap::Parser; -use libsql_storage::rpc::storage_server::{Storage, StorageServer}; -use libsql_storage::rpc::{ - DbSizeRequest, DbSizeResponse, DestroyRequest, DestroyResponse, FindFrameRequest, - FindFrameResponse, FramePageNumRequest, FramePageNumResponse, FramesInWalRequest, - FramesInWalResponse, InsertFramesRequest, InsertFramesResponse, ReadFrameRequest, - ReadFrameResponse, -}; +use libsql_storage::rpc::storage_server::StorageServer; use libsql_storage_server::version::Version; -use redis::{Client, Commands, RedisResult}; -use serde; +use redis::Client; use service::Service; use tokio::sync::Mutex; -use tonic::{transport::Server, Request, Response, Status}; -use tracing::{error, trace}; +use tonic::transport::Server; +use tracing::trace; use tracing_subscriber::{EnvFilter, FmtSubscriber}; -use crate::fdb_store::FDBFrameStore; -use crate::memory_store::InMemFrameStore; -use crate::redis_store::RedisFrameStore; -use crate::store::FrameStore; - mod fdb_store; mod memory_store; mod redis_store; @@ -41,12 +29,13 @@ enum StorageType { #[derive(Debug, Parser)] #[command(name = "libsql-storage-server")] -#[command(about = "libSQL storage server", version = Version::default(), long_about = None)] +#[command(about = "libSQL Storage Server", version = Version::default(), long_about = None)] struct Cli { /// The address and port the storage RPC protocol listens to. Example: `127.0.0.1:5002`. #[clap(long, env = "LIBSQL_STORAGE_LISTEN_ADDR", default_value = "[::]:5002")] listen_addr: SocketAddr, + /// The type of storage backend to use. Example: `redis` #[clap(value_enum, long, default_value = "in-memory")] storage_type: StorageType, } @@ -61,18 +50,28 @@ async fn main() -> Result<()> { .expect("setting default subscriber failed"); let args = Cli::parse(); - // export REDIS_ADDR=http://libsql-storage-server.internal:5002 - let redis_addr = std::env::var("REDIS_ADDR").unwrap_or("redis://127.0.0.1/".to_string()); - let client = Client::open(redis_addr).unwrap(); - let service = Service::with_store(Arc::new(Mutex::new(FDBFrameStore::new()))); + let service = match args.storage_type { + StorageType::Redis => { + // export REDIS_ADDR=http://libsql-storage-server.internal:5002 + let redis_addr = + std::env::var("REDIS_ADDR").unwrap_or("redis://127.0.0.1/".to_string()); + let client = Client::open(redis_addr).unwrap(); + Service::with_store(Arc::new(Mutex::new(RedisFrameStore::new(client)))) + } + StorageType::FoundationDB => { + Service::with_store(Arc::new(Mutex::new(FDBFrameStore::new()))) + } + _ => Service::new(), + }; + trace!( - "(trace) Starting libSQL storage server on {}", + "Starting libSQL storage server (with type {:?}) on {}", + args.storage_type, args.listen_addr ); Server::builder() .add_service(StorageServer::new(service)) .serve(args.listen_addr) .await?; - Ok(()) } diff --git a/libsql-storage-server/src/service.rs b/libsql-storage-server/src/service.rs index dc2da90ef9..862ef9bbaf 100644 --- a/libsql-storage-server/src/service.rs +++ b/libsql-storage-server/src/service.rs @@ -1,13 +1,7 @@ use crate::memory_store::InMemFrameStore; use crate::store::{FrameData, FrameStore}; -use bytes::Bytes; +use libsql_storage::rpc; use libsql_storage::rpc::storage_server::Storage; -use libsql_storage::rpc::{ - DbSizeRequest, DbSizeResponse, DestroyRequest, DestroyResponse, FindFrameRequest, - FindFrameResponse, FramePageNumRequest, FramePageNumResponse, FramesInWalRequest, - FramesInWalResponse, InsertFramesRequest, InsertFramesResponse, ReadFrameRequest, - ReadFrameResponse, -}; use std::sync::atomic::AtomicU32; use std::sync::Arc; use tokio::sync::Mutex; @@ -38,8 +32,8 @@ impl Service { impl Storage for Service { async fn insert_frames( &self, - request: tonic::Request, - ) -> anyhow::Result, tonic::Status> { + request: tonic::Request, + ) -> anyhow::Result, tonic::Status> { trace!("insert_frames()"); let mut num_frames = 0; let mut store = self.store.lock().await; @@ -69,13 +63,13 @@ impl Storage for Service { self.db_size .fetch_add(1, std::sync::atomic::Ordering::SeqCst); } - Ok(Response::new(InsertFramesResponse { num_frames })) + Ok(Response::new(rpc::InsertFramesResponse { num_frames })) } async fn find_frame( &self, - request: tonic::Request, - ) -> anyhow::Result, tonic::Status> { + request: tonic::Request, + ) -> anyhow::Result, tonic::Status> { let request = request.into_inner(); let page_no = request.page_no; let namespace = request.namespace; @@ -87,19 +81,19 @@ impl Storage for Service { .find_frame(&namespace, page_no) .await { - Ok(Response::new(FindFrameResponse { + Ok(Response::new(rpc::FindFrameResponse { frame_no: Some(frame_no), })) } else { error!("find_frame() failed for page_no={}", page_no); - Ok(Response::new(FindFrameResponse { frame_no: None })) + Ok(Response::new(rpc::FindFrameResponse { frame_no: None })) } } async fn read_frame( &self, - request: tonic::Request, - ) -> anyhow::Result, tonic::Status> { + request: tonic::Request, + ) -> anyhow::Result, tonic::Status> { let request = request.into_inner(); let frame_no = request.frame_no; let namespace = request.namespace; @@ -111,37 +105,37 @@ impl Storage for Service { .read_frame(&namespace, frame_no) .await { - Ok(Response::new(ReadFrameResponse { + Ok(Response::new(rpc::ReadFrameResponse { frame: Some(data.clone().into()), })) } else { error!("read_frame() failed for frame_no={}", frame_no); - Ok(Response::new(ReadFrameResponse { frame: None })) + Ok(Response::new(rpc::ReadFrameResponse { frame: None })) } } async fn db_size( &self, - request: tonic::Request, - ) -> anyhow::Result, tonic::Status> { + request: tonic::Request, + ) -> anyhow::Result, tonic::Status> { let size = self.db_size.load(std::sync::atomic::Ordering::SeqCst) as u64; - Ok(Response::new(DbSizeResponse { size })) + Ok(Response::new(rpc::DbSizeResponse { size })) } async fn frames_in_wal( &self, - request: Request, - ) -> std::result::Result, Status> { + request: Request, + ) -> std::result::Result, Status> { let namespace = request.into_inner().namespace; - Ok(Response::new(FramesInWalResponse { + Ok(Response::new(rpc::FramesInWalResponse { count: self.store.lock().await.frames_in_wal(&namespace).await, })) } async fn frame_page_num( &self, - request: Request, - ) -> std::result::Result, Status> { + request: Request, + ) -> std::result::Result, Status> { let request = request.into_inner(); let frame_no = request.frame_no; let namespace = request.namespace; @@ -152,20 +146,20 @@ impl Storage for Service { .frame_page_no(&namespace, frame_no) .await { - Ok(Response::new(FramePageNumResponse { page_no })) + Ok(Response::new(rpc::FramePageNumResponse { page_no })) } else { error!("frame_page_num() failed for frame_no={}", frame_no); - Ok(Response::new(FramePageNumResponse { page_no: 0 })) + Ok(Response::new(rpc::FramePageNumResponse { page_no: 0 })) } } async fn destroy( &self, - request: tonic::Request, - ) -> anyhow::Result, tonic::Status> { + request: tonic::Request, + ) -> anyhow::Result, tonic::Status> { trace!("destroy()"); let namespace = request.into_inner().namespace; self.store.lock().await.destroy(&namespace).await; - Ok(Response::new(DestroyResponse {})) + Ok(Response::new(rpc::DestroyResponse {})) } } From 5f73b164b55f73327453be440f7be06caa880c81 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Fri, 31 May 2024 21:41:56 +0530 Subject: [PATCH 46/60] moar ref --- libsql-storage-server/src/fdb_store.rs | 8 ++-- libsql-storage-server/src/main.rs | 12 +++--- libsql-storage-server/src/memory_store.rs | 17 +++++---- libsql-storage-server/src/redis_store.rs | 4 +- libsql-storage-server/src/service.rs | 46 +++++++++-------------- 5 files changed, 38 insertions(+), 49 deletions(-) diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index c2784b449d..1a5cb3fabf 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -8,13 +8,13 @@ use foundationdb::Transaction; use tracing::error; pub struct FDBFrameStore { - network: NetworkAutoStop, + _network: NetworkAutoStop, } impl FDBFrameStore { pub fn new() -> Self { - let network = unsafe { foundationdb::boot() }; - Self { network } + let _network = unsafe { foundationdb::boot() }; + Self { _network } } async fn get_max_frame_no(&self, txn: &Transaction, namespace: &str) -> u64 { @@ -149,5 +149,5 @@ impl FrameStore for FDBFrameStore { self.get_max_frame_no(&txn, namespace).await } - async fn destroy(&mut self, namespace: &str) {} + async fn destroy(&mut self, _namespace: &str) {} } diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 7eea0cf592..baaa8d08bc 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -1,3 +1,9 @@ +mod fdb_store; +mod memory_store; +mod redis_store; +mod service; +mod store; + use std::net::SocketAddr; use std::sync::Arc; @@ -14,12 +20,6 @@ use tonic::transport::Server; use tracing::trace; use tracing_subscriber::{EnvFilter, FmtSubscriber}; -mod fdb_store; -mod memory_store; -mod redis_store; -mod service; -mod store; - #[derive(clap::ValueEnum, Clone, Debug)] enum StorageType { InMemory, diff --git a/libsql-storage-server/src/memory_store.rs b/libsql-storage-server/src/memory_store.rs index 17ec0d17d8..8f1bf6fbb9 100644 --- a/libsql-storage-server/src/memory_store.rs +++ b/libsql-storage-server/src/memory_store.rs @@ -1,8 +1,9 @@ +use std::collections::BTreeMap; + use crate::store::FrameData; use crate::store::FrameStore; use async_trait::async_trait; use bytes::Bytes; -use std::collections::BTreeMap; #[derive(Default)] pub(crate) struct InMemFrameStore { @@ -22,7 +23,7 @@ impl InMemFrameStore { #[async_trait] impl FrameStore for InMemFrameStore { // inserts a new frame for the page number and returns the new frame value - async fn insert_frame(&mut self, namespace: &str, page_no: u64, frame: Bytes) -> u64 { + async fn insert_frame(&mut self, _namespace: &str, page_no: u64, frame: Bytes) -> u64 { let frame_no = self.max_frame_no + 1; self.max_frame_no = frame_no; self.frames.insert( @@ -39,31 +40,31 @@ impl FrameStore for InMemFrameStore { frame_no } - async fn insert_frames(&mut self, namespace: &str, frames: Vec) -> u64 { + async fn insert_frames(&mut self, _namespace: &str, _frames: Vec) -> u64 { todo!() } - async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option { + async fn read_frame(&self, _namespace: &str, frame_no: u64) -> Option { self.frames.get(&frame_no).map(|frame| frame.data.clone()) } // given a page number, return the maximum frame for the page - async fn find_frame(&self, namespace: &str, page_no: u64) -> Option { + async fn find_frame(&self, _namespace: &str, page_no: u64) -> Option { self.pages .get(&page_no) .map(|frames| *frames.last().unwrap()) } // given a frame num, return the page number - async fn frame_page_no(&self, namespace: &str, frame_no: u64) -> Option { + async fn frame_page_no(&self, _namespace: &str, frame_no: u64) -> Option { self.frames.get(&frame_no).map(|frame| frame.page_no) } - async fn frames_in_wal(&self, namespace: &str) -> u64 { + async fn frames_in_wal(&self, _namespace: &str) -> u64 { self.max_frame_no } - async fn destroy(&mut self, namespace: &str) { + async fn destroy(&mut self, _namespace: &str) { self.frames.clear(); self.pages.clear(); self.max_frame_no = 0; diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index 43c9fc67a5..7f6c95d1d1 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -56,7 +56,7 @@ impl FrameStore for RedisFrameStore { max_frame_no } - async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option { + async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option { let frame_key = format!("f/{}/{}", namespace, frame_no); let mut con = self.client.get_connection().unwrap(); let result = con.hget::>(frame_key.clone(), "f"); @@ -119,7 +119,7 @@ impl FrameStore for RedisFrameStore { }) } - async fn destroy(&mut self, namespace: &str) { + async fn destroy(&mut self, _namespace: &str) { // remove all the keys in redis let mut con = self.client.get_connection().unwrap(); // send a FLUSHALL request diff --git a/libsql-storage-server/src/service.rs b/libsql-storage-server/src/service.rs index 862ef9bbaf..bc7237a3f9 100644 --- a/libsql-storage-server/src/service.rs +++ b/libsql-storage-server/src/service.rs @@ -1,9 +1,10 @@ +use std::sync::atomic::AtomicU32; +use std::sync::Arc; + use crate::memory_store::InMemFrameStore; -use crate::store::{FrameData, FrameStore}; +use crate::store::FrameStore; use libsql_storage::rpc; use libsql_storage::rpc::storage_server::Storage; -use std::sync::atomic::AtomicU32; -use std::sync::Arc; use tokio::sync::Mutex; use tonic::{Request, Response, Status}; use tracing::{error, trace}; @@ -32,26 +33,13 @@ impl Service { impl Storage for Service { async fn insert_frames( &self, - request: tonic::Request, - ) -> anyhow::Result, tonic::Status> { - trace!("insert_frames()"); + request: Request, + ) -> anyhow::Result, Status> { let mut num_frames = 0; let mut store = self.store.lock().await; - trace!("insert_frames() got lock"); let request = request.into_inner(); let namespace = request.namespace; - let frames = request.frames.into_iter().map(|frame| FrameData { - page_no: frame.page_no, - data: frame.data.into(), - }); - let all_data: Vec = frames - .clone() - .map(|f| f.data.clone().to_vec()) - .flatten() - .collect(); - trace!("insert_frames() got frames (bytes): {:?}", all_data.len()); - trace!("insert_frames() got frames: {:?}", frames.len()); - for frame in frames { + for frame in request.frames.into_iter() { trace!( "inserted for page {} frame {}", frame.page_no, @@ -68,8 +56,8 @@ impl Storage for Service { async fn find_frame( &self, - request: tonic::Request, - ) -> anyhow::Result, tonic::Status> { + request: Request, + ) -> Result, Status> { let request = request.into_inner(); let page_no = request.page_no; let namespace = request.namespace; @@ -92,8 +80,8 @@ impl Storage for Service { async fn read_frame( &self, - request: tonic::Request, - ) -> anyhow::Result, tonic::Status> { + request: Request, + ) -> Result, Status> { let request = request.into_inner(); let frame_no = request.frame_no; let namespace = request.namespace; @@ -116,8 +104,8 @@ impl Storage for Service { async fn db_size( &self, - request: tonic::Request, - ) -> anyhow::Result, tonic::Status> { + _request: Request, + ) -> Result, Status> { let size = self.db_size.load(std::sync::atomic::Ordering::SeqCst) as u64; Ok(Response::new(rpc::DbSizeResponse { size })) } @@ -125,7 +113,7 @@ impl Storage for Service { async fn frames_in_wal( &self, request: Request, - ) -> std::result::Result, Status> { + ) -> Result, Status> { let namespace = request.into_inner().namespace; Ok(Response::new(rpc::FramesInWalResponse { count: self.store.lock().await.frames_in_wal(&namespace).await, @@ -135,7 +123,7 @@ impl Storage for Service { async fn frame_page_num( &self, request: Request, - ) -> std::result::Result, Status> { + ) -> Result, Status> { let request = request.into_inner(); let frame_no = request.frame_no; let namespace = request.namespace; @@ -155,8 +143,8 @@ impl Storage for Service { async fn destroy( &self, - request: tonic::Request, - ) -> anyhow::Result, tonic::Status> { + request: Request, + ) -> Result, Status> { trace!("destroy()"); let namespace = request.into_inner().namespace; self.store.lock().await.destroy(&namespace).await; From f5fa261406a877a0cff7653d907e9ee68b136440 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Fri, 31 May 2024 21:47:01 +0530 Subject: [PATCH 47/60] Refactor --- libsql-storage-server/src/service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsql-storage-server/src/service.rs b/libsql-storage-server/src/service.rs index bc7237a3f9..a4147ca260 100644 --- a/libsql-storage-server/src/service.rs +++ b/libsql-storage-server/src/service.rs @@ -34,7 +34,7 @@ impl Storage for Service { async fn insert_frames( &self, request: Request, - ) -> anyhow::Result, Status> { + ) -> Result, Status> { let mut num_frames = 0; let mut store = self.store.lock().await; let request = request.into_inner(); From 601d1a2d57597cf9747d96b924c979d39f2cbc39 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 1 Jun 2024 21:14:46 +0530 Subject: [PATCH 48/60] do not use mutex for clinet --- libsql-storage/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 47ad5a25f8..90595684fc 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -89,7 +89,7 @@ impl WalManager for DurableWalManager { pub struct DurableWal { namespace: String, conn_id: String, - client: parking_lot::Mutex>, + client: StorageClient, frames_cache: SieveCache>, write_cache: BTreeMap, lock_manager: Arc>, @@ -118,7 +118,7 @@ impl DurableWal { Self { namespace, conn_id: uuid::Uuid::new_v4().to_string(), - client: parking_lot::Mutex::new(client), + client, frames_cache: page_frames, write_cache: BTreeMap::new(), lock_manager, @@ -137,7 +137,7 @@ impl DurableWal { page_no: page_no.get() as u64, max_frame_no: 0, }; - let mut binding = self.client.lock(); + let mut binding = self.client.clone(); let resp = binding.find_frame(req); let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); let frame_no = resp @@ -152,7 +152,7 @@ impl DurableWal { let req = rpc::FramesInWalRequest { namespace: self.namespace.clone(), }; - let mut binding = self.client.lock(); + let mut binding = self.client.clone(); let resp = binding.frames_in_wal(req); let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); let count = resp.into_inner().count; @@ -226,7 +226,7 @@ impl Wal for DurableWal { namespace: self.namespace.clone(), frame_no: frame_no.get(), }; - let mut binding = self.client.lock(); + let mut binding = self.client.clone(); let resp = binding.read_frame(req); let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); let frame = resp.into_inner().frame.unwrap(); @@ -323,7 +323,7 @@ impl Wal for DurableWal { max_frame_no: 0, }; self.write_cache.clear(); - let mut binding = self.client.lock(); + let mut binding = self.client.clone(); trace!("sending DurableWal::insert_frames() {:?}", req.frames.len()); let resp = binding.insert_frames(req); let resp = tokio::task::block_in_place(|| self.rt.block_on(resp)).unwrap(); From a6428098d1c64e440ac4e9e1e775b7393430f68a Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 1 Jun 2024 21:36:03 +0530 Subject: [PATCH 49/60] Add storage addr configurable --- libsql-server/src/connection/libsql.rs | 7 +++++- libsql-storage/src/lib.rs | 33 +++++++++++++++++++------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/libsql-server/src/connection/libsql.rs b/libsql-server/src/connection/libsql.rs index 7ec7e22427..d8e1e903f8 100644 --- a/libsql-server/src/connection/libsql.rs +++ b/libsql-server/src/connection/libsql.rs @@ -321,7 +321,12 @@ where move || -> crate::Result<_> { let manager = ManagedConnectionWalWrapper::new(connection_manager); let id = manager.id(); - let durable_wal = DurableWalManager::new(lock_manager); + + // // connect to external storage server + // // export LIBSQL_STORAGE_SERVER_ADDR=http://libsql-storage-server.internal:5002 + let address = std::env::var("LIBSQL_STORAGE_SERVER_ADDR") + .unwrap_or("http://127.0.0.1:5002".to_string()); + let durable_wal = DurableWalManager::new(lock_manager, address); let wal = durable_wal.wrap(manager).wrap(wal_wrapper); let conn = Connection::new( diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 90595684fc..6882cd05c9 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -22,14 +22,25 @@ pub mod rpc { // - no multi tenancy, uses `default` namespace // - txn can read new frames after it started (since there are no read locks) +#[derive(Clone, Default)] +pub struct DurableWalConfig { + storage_server_address: String, +} + #[derive(Clone)] pub struct DurableWalManager { lock_manager: Arc>, + config: DurableWalConfig, } impl DurableWalManager { - pub fn new(lock_manager: Arc>) -> Self { - Self { lock_manager } + pub fn new(lock_manager: Arc>, storage_server_address: String) -> Self { + Self { + lock_manager, + config: DurableWalConfig { + storage_server_address, + }, + } } } @@ -53,7 +64,11 @@ impl WalManager for DurableWalManager { trace!("DurableWalManager::open(db_path: {})", db_path); // TODO: use the actual namespace uuid from the connection let namespace = "default".to_string(); - Ok(DurableWal::new(namespace, self.lock_manager.clone())) + Ok(DurableWal::new( + namespace, + self.config.clone(), + self.lock_manager.clone(), + )) } fn close( @@ -98,7 +113,11 @@ pub struct DurableWal { } impl DurableWal { - fn new(namespace: String, lock_manager: Arc>) -> Self { + fn new( + namespace: String, + config: DurableWalConfig, + lock_manager: Arc>, + ) -> Self { let (_runtime, rt) = match tokio::runtime::Handle::try_current() { Ok(h) => (None, h), Err(_) => { @@ -107,11 +126,7 @@ impl DurableWal { (Some(rt), handle) } }; - // connect to external storage server - // export LIBSQL_STORAGE_SERVER_ADDR=http://libsql-storage-server.internal:5002 - let address = std::env::var("LIBSQL_STORAGE_SERVER_ADDR") - .unwrap_or("http://127.0.0.1:5002".to_string()); - let client = StorageClient::connect(address); + let client = StorageClient::connect(config.storage_server_address); let client = tokio::task::block_in_place(|| rt.block_on(client)).unwrap(); let page_frames = SieveCache::new(1000).unwrap(); From ef8b4bf7c158c851e01ede16b2a5a1c644794000 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sun, 2 Jun 2024 04:45:33 +0530 Subject: [PATCH 50/60] Update trait to take non mutable ref --- libsql-storage-server/src/fdb_store.rs | 8 ++-- libsql-storage-server/src/main.rs | 6 +-- libsql-storage-server/src/memory_store.rs | 50 ++++++++++++++++------- libsql-storage-server/src/redis_store.rs | 6 +-- libsql-storage-server/src/service.rs | 38 +++++------------ libsql-storage-server/src/store.rs | 6 +-- 6 files changed, 58 insertions(+), 56 deletions(-) diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index 1a5cb3fabf..59301bda56 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -34,7 +34,7 @@ impl FDBFrameStore { } async fn insert_with_tx( - &mut self, + &self, namespace: &str, txn: &Transaction, frame_no: u64, @@ -52,7 +52,7 @@ impl FDBFrameStore { #[async_trait] impl FrameStore for FDBFrameStore { - async fn insert_frame(&mut self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64 { + async fn insert_frame(&self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64 { let max_frame_key = format!("{}/max_frame_no", namespace); let db = foundationdb::Database::default().unwrap(); let txn = db.create_trx().expect("unable to create transaction"); @@ -72,7 +72,7 @@ impl FrameStore for FDBFrameStore { frame_no } - async fn insert_frames(&mut self, namespace: &str, frames: Vec) -> u64 { + async fn insert_frames(&self, namespace: &str, frames: Vec) -> u64 { let max_frame_key = format!("{}/max_frame_no", namespace); let db = foundationdb::Database::default().unwrap(); let txn = db.create_trx().expect("unable to create transaction"); @@ -149,5 +149,5 @@ impl FrameStore for FDBFrameStore { self.get_max_frame_no(&txn, namespace).await } - async fn destroy(&mut self, _namespace: &str) {} + async fn destroy(&self, _namespace: &str) {} } diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index baaa8d08bc..62e4df3664 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -56,11 +56,9 @@ async fn main() -> Result<()> { let redis_addr = std::env::var("REDIS_ADDR").unwrap_or("redis://127.0.0.1/".to_string()); let client = Client::open(redis_addr).unwrap(); - Service::with_store(Arc::new(Mutex::new(RedisFrameStore::new(client)))) - } - StorageType::FoundationDB => { - Service::with_store(Arc::new(Mutex::new(FDBFrameStore::new()))) + Service::with_store(Box::new(RedisFrameStore::new(client))) } + StorageType::FoundationDB => Service::with_store(Box::new(FDBFrameStore::new())), _ => Service::new(), }; diff --git a/libsql-storage-server/src/memory_store.rs b/libsql-storage-server/src/memory_store.rs index 8f1bf6fbb9..f31ee43d82 100644 --- a/libsql-storage-server/src/memory_store.rs +++ b/libsql-storage-server/src/memory_store.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::sync::Mutex; use crate::store::FrameData; use crate::store::FrameStore; @@ -7,6 +8,11 @@ use bytes::Bytes; #[derive(Default)] pub(crate) struct InMemFrameStore { + inner: Mutex, +} + +#[derive(Default)] +struct InMemInternal { // contains a frame data, key is the frame number frames: BTreeMap, // pages map contains the page number as a key and the list of frames for the page as a value @@ -23,50 +29,66 @@ impl InMemFrameStore { #[async_trait] impl FrameStore for InMemFrameStore { // inserts a new frame for the page number and returns the new frame value - async fn insert_frame(&mut self, _namespace: &str, page_no: u64, frame: Bytes) -> u64 { - let frame_no = self.max_frame_no + 1; - self.max_frame_no = frame_no; - self.frames.insert( + async fn insert_frame(&self, _namespace: &str, page_no: u64, frame: Bytes) -> u64 { + let mut inner = self.inner.lock().unwrap(); + let frame_no = inner.max_frame_no + 1; + inner.max_frame_no = frame_no; + inner.frames.insert( frame_no, FrameData { page_no, data: frame, }, ); - self.pages + inner + .pages .entry(page_no) .or_insert_with(Vec::new) .push(frame_no); frame_no } - async fn insert_frames(&mut self, _namespace: &str, _frames: Vec) -> u64 { + async fn insert_frames(&self, _namespace: &str, _frames: Vec) -> u64 { todo!() } async fn read_frame(&self, _namespace: &str, frame_no: u64) -> Option { - self.frames.get(&frame_no).map(|frame| frame.data.clone()) + self.inner + .lock() + .unwrap() + .frames + .get(&frame_no) + .map(|frame| frame.data.clone()) } // given a page number, return the maximum frame for the page async fn find_frame(&self, _namespace: &str, page_no: u64) -> Option { - self.pages + self.inner + .lock() + .unwrap() + .pages .get(&page_no) .map(|frames| *frames.last().unwrap()) } // given a frame num, return the page number async fn frame_page_no(&self, _namespace: &str, frame_no: u64) -> Option { - self.frames.get(&frame_no).map(|frame| frame.page_no) + self.inner + .lock() + .unwrap() + .frames + .get(&frame_no) + .map(|frame| frame.page_no) } async fn frames_in_wal(&self, _namespace: &str) -> u64 { - self.max_frame_no + self.inner.lock().unwrap().max_frame_no } - async fn destroy(&mut self, _namespace: &str) { - self.frames.clear(); - self.pages.clear(); - self.max_frame_no = 0; + async fn destroy(&self, _namespace: &str) { + let mut inner = self.inner.lock().unwrap(); + inner.frames.clear(); + inner.pages.clear(); + inner.max_frame_no = 0; } } diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index 7f6c95d1d1..31107f3dd6 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -17,7 +17,7 @@ impl RedisFrameStore { #[async_trait] impl FrameStore for RedisFrameStore { - async fn insert_frame(&mut self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64 { + async fn insert_frame(&self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64 { let max_frame_key = format!("{}/max_frame_no", namespace); let mut con = self.client.get_connection().unwrap(); @@ -48,7 +48,7 @@ impl FrameStore for RedisFrameStore { max_frame_no } - async fn insert_frames(&mut self, namespace: &str, frames: Vec) -> u64 { + async fn insert_frames(&self, namespace: &str, frames: Vec) -> u64 { let mut max_frame_no = 0; for f in frames { max_frame_no = self.insert_frame(namespace, f.page_no, f.data).await; @@ -119,7 +119,7 @@ impl FrameStore for RedisFrameStore { }) } - async fn destroy(&mut self, _namespace: &str) { + async fn destroy(&self, _namespace: &str) { // remove all the keys in redis let mut con = self.client.get_connection().unwrap(); // send a FLUSHALL request diff --git a/libsql-storage-server/src/service.rs b/libsql-storage-server/src/service.rs index a4147ca260..e0b54b6fbe 100644 --- a/libsql-storage-server/src/service.rs +++ b/libsql-storage-server/src/service.rs @@ -2,6 +2,7 @@ use std::sync::atomic::AtomicU32; use std::sync::Arc; use crate::memory_store::InMemFrameStore; +use crate::service; use crate::store::FrameStore; use libsql_storage::rpc; use libsql_storage::rpc::storage_server::Storage; @@ -10,18 +11,18 @@ use tonic::{Request, Response, Status}; use tracing::{error, trace}; pub struct Service { - store: Arc>, + store: Box, db_size: AtomicU32, } impl Service { pub fn new() -> Self { Self { - store: Arc::new(Mutex::new(InMemFrameStore::new())), + store: Box::new(InMemFrameStore::new()), db_size: AtomicU32::new(0), } } - pub fn with_store(store: Arc>) -> Self { + pub fn with_store(store: Box) -> Self { Self { store, db_size: AtomicU32::new(0), @@ -36,14 +37,13 @@ impl Storage for Service { request: Request, ) -> Result, Status> { let mut num_frames = 0; - let mut store = self.store.lock().await; let request = request.into_inner(); let namespace = request.namespace; for frame in request.frames.into_iter() { trace!( "inserted for page {} frame {}", frame.page_no, - store + self.store .insert_frame(&namespace, frame.page_no, frame.data.into()) .await ); @@ -62,13 +62,7 @@ impl Storage for Service { let page_no = request.page_no; let namespace = request.namespace; trace!("find_frame(page_no={})", page_no); - if let Some(frame_no) = self - .store - .lock() - .await - .find_frame(&namespace, page_no) - .await - { + if let Some(frame_no) = self.store.find_frame(&namespace, page_no).await { Ok(Response::new(rpc::FindFrameResponse { frame_no: Some(frame_no), })) @@ -86,13 +80,7 @@ impl Storage for Service { let frame_no = request.frame_no; let namespace = request.namespace; trace!("read_frame(frame_no={})", frame_no); - if let Some(data) = self - .store - .lock() - .await - .read_frame(&namespace, frame_no) - .await - { + if let Some(data) = self.store.read_frame(&namespace, frame_no).await { Ok(Response::new(rpc::ReadFrameResponse { frame: Some(data.clone().into()), })) @@ -116,7 +104,7 @@ impl Storage for Service { ) -> Result, Status> { let namespace = request.into_inner().namespace; Ok(Response::new(rpc::FramesInWalResponse { - count: self.store.lock().await.frames_in_wal(&namespace).await, + count: self.store.frames_in_wal(&namespace).await, })) } @@ -127,13 +115,7 @@ impl Storage for Service { let request = request.into_inner(); let frame_no = request.frame_no; let namespace = request.namespace; - if let Some(page_no) = self - .store - .lock() - .await - .frame_page_no(&namespace, frame_no) - .await - { + if let Some(page_no) = self.store.frame_page_no(&namespace, frame_no).await { Ok(Response::new(rpc::FramePageNumResponse { page_no })) } else { error!("frame_page_num() failed for frame_no={}", frame_no); @@ -147,7 +129,7 @@ impl Storage for Service { ) -> Result, Status> { trace!("destroy()"); let namespace = request.into_inner().namespace; - self.store.lock().await.destroy(&namespace).await; + self.store.destroy(&namespace).await; Ok(Response::new(rpc::DestroyResponse {})) } } diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index fcd38835b0..3445bed8be 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -3,13 +3,13 @@ use bytes::Bytes; #[async_trait] pub trait FrameStore: Send + Sync { - async fn insert_frame(&mut self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64; - async fn insert_frames(&mut self, namespace: &str, frames: Vec) -> u64; + async fn insert_frame(&self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64; + async fn insert_frames(&self, namespace: &str, frames: Vec) -> u64; async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option; async fn find_frame(&self, namespace: &str, page_no: u64) -> Option; async fn frame_page_no(&self, namespace: &str, frame_no: u64) -> Option; async fn frames_in_wal(&self, namespace: &str) -> u64; - async fn destroy(&mut self, namespace: &str); + async fn destroy(&self, namespace: &str); } #[derive(Default)] From 1d5355a05d9674108f20c848b818d09c62f9544e Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sun, 2 Jun 2024 04:47:36 +0530 Subject: [PATCH 51/60] cargo fix --- libsql-storage-server/src/main.rs | 2 -- libsql-storage-server/src/service.rs | 3 --- libsql-storage-server/src/store.rs | 1 + 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 62e4df3664..d9b473b879 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -5,7 +5,6 @@ mod service; mod store; use std::net::SocketAddr; -use std::sync::Arc; use crate::fdb_store::FDBFrameStore; use crate::redis_store::RedisFrameStore; @@ -15,7 +14,6 @@ use libsql_storage::rpc::storage_server::StorageServer; use libsql_storage_server::version::Version; use redis::Client; use service::Service; -use tokio::sync::Mutex; use tonic::transport::Server; use tracing::trace; use tracing_subscriber::{EnvFilter, FmtSubscriber}; diff --git a/libsql-storage-server/src/service.rs b/libsql-storage-server/src/service.rs index e0b54b6fbe..eeae97eb48 100644 --- a/libsql-storage-server/src/service.rs +++ b/libsql-storage-server/src/service.rs @@ -1,12 +1,9 @@ use std::sync::atomic::AtomicU32; -use std::sync::Arc; use crate::memory_store::InMemFrameStore; -use crate::service; use crate::store::FrameStore; use libsql_storage::rpc; use libsql_storage::rpc::storage_server::Storage; -use tokio::sync::Mutex; use tonic::{Request, Response, Status}; use tracing::{error, trace}; diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index 3445bed8be..e05e64c3bb 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -4,6 +4,7 @@ use bytes::Bytes; #[async_trait] pub trait FrameStore: Send + Sync { async fn insert_frame(&self, namespace: &str, page_no: u64, frame: bytes::Bytes) -> u64; + #[allow(dead_code)] async fn insert_frames(&self, namespace: &str, frames: Vec) -> u64; async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option; async fn find_frame(&self, namespace: &str, page_no: u64) -> Option; From 38cb30f6e492c7034fb156c441e2381c7698216e Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Thu, 6 Jun 2024 18:34:39 +0530 Subject: [PATCH 52/60] send max frame num --- libsql-storage-server/src/fdb_store.rs | 34 ++++++++++++++++++++--- libsql-storage-server/src/main.rs | 1 + libsql-storage-server/src/memory_store.rs | 2 +- libsql-storage-server/src/redis_store.rs | 2 +- libsql-storage-server/src/service.rs | 7 +++-- libsql-storage-server/src/store.rs | 2 +- libsql-storage/src/lib.rs | 5 +++- 7 files changed, 43 insertions(+), 10 deletions(-) diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index 7f753de61c..729bf293c5 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use foundationdb::api::NetworkAutoStop; use foundationdb::tuple::pack; use foundationdb::tuple::unpack; -use foundationdb::Transaction; +use foundationdb::{KeySelector, Transaction}; use tracing::error; pub struct FDBFrameStore { @@ -13,6 +13,7 @@ pub struct FDBFrameStore { impl FDBFrameStore { pub fn new() -> Self { + println!("I was called"); let _network = unsafe { foundationdb::boot() }; Self { _network } } @@ -43,10 +44,12 @@ impl FDBFrameStore { let frame_data_key = format!("{}/f/{}/f", namespace, frame_no); let frame_page_key = format!("{}/f/{}/p", namespace, frame_no); let page_key = format!("{}/p/{}", namespace, frame.page_no); + let page_frame_idx = format!("{}/pf/{}/{}", namespace, frame.page_no, frame_no); txn.set(&frame_data_key.as_bytes(), &frame.data); txn.set(&frame_page_key.as_bytes(), &pack(&frame.page_no)); txn.set(&page_key.as_bytes(), &pack(&frame_no)); + txn.set(&page_frame_idx.as_bytes(), &pack(&frame_no)); } } @@ -108,12 +111,34 @@ impl FrameStore for FDBFrameStore { None } - async fn find_frame(&self, namespace: &str, page_no: u32) -> Option { - let page_key = format!("{}/p/{}", namespace, page_no); - + async fn find_frame(&self, namespace: &str, page_no: u32, max_frame_no: u64) -> Option { let db = foundationdb::Database::default().unwrap(); let txn = db.create_trx().expect("unable to create transaction"); + let page_key = format!("{}/pf/{}/{}", namespace, page_no, max_frame_no); + let result = txn + .get( + KeySelector::last_less_or_equal(page_key.as_bytes()).key(), + false, + ) + .await; + // if let Err(e) = result { + // error!("get failed: {:?}", e); + // return None; + // } + // if let Ok(None) = result { + // error!("page not found (with max)"); + // return None; + // } + if let Ok(result) = result { + if let Some(frame_no) = result { + let frame_no: u64 = unpack(&frame_no).expect("failed to decode u64"); + tracing::trace!("got the frame_no = {} (with max)", frame_no); + }; + }; + + let page_key = format!("{}/p/{}", namespace, page_no); + let result = txn.get(&page_key.as_bytes(), false).await; if let Err(e) = result { error!("get failed: {:?}", e); @@ -124,6 +149,7 @@ impl FrameStore for FDBFrameStore { return None; } let frame_no: u64 = unpack(&result.unwrap().unwrap()).expect("failed to decode u64"); + tracing::trace!("got the frame_no = {} (without max)", frame_no); Some(frame_no) } diff --git a/libsql-storage-server/src/main.rs b/libsql-storage-server/src/main.rs index 6a22db589f..0e1e1fdc6b 100644 --- a/libsql-storage-server/src/main.rs +++ b/libsql-storage-server/src/main.rs @@ -22,6 +22,7 @@ use tracing_subscriber::{EnvFilter, FmtSubscriber}; enum StorageType { InMemory, Redis, + #[cfg(feature = "foundation-db")] FoundationDB, } diff --git a/libsql-storage-server/src/memory_store.rs b/libsql-storage-server/src/memory_store.rs index 41cd427b14..952c39574d 100644 --- a/libsql-storage-server/src/memory_store.rs +++ b/libsql-storage-server/src/memory_store.rs @@ -61,7 +61,7 @@ impl FrameStore for InMemFrameStore { } // given a page number, return the maximum frame for the page - async fn find_frame(&self, _namespace: &str, page_no: u32) -> Option { + async fn find_frame(&self, _namespace: &str, page_no: u32, _max_frame_no: u64) -> Option { self.inner .lock() .unwrap() diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index 14f32903dc..33379ea9d7 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -75,7 +75,7 @@ impl FrameStore for RedisFrameStore { } } - async fn find_frame(&self, namespace: &str, page_no: u32) -> Option { + async fn find_frame(&self, namespace: &str, page_no: u32, _max_frame_no: u64) -> Option { let page_key = format!("p/{}/{}", namespace, page_no); let mut con = self.client.get_connection().unwrap(); let frame_no = con.get::(page_key.clone()); diff --git a/libsql-storage-server/src/service.rs b/libsql-storage-server/src/service.rs index 27183b2dc5..1b0e13009d 100644 --- a/libsql-storage-server/src/service.rs +++ b/libsql-storage-server/src/service.rs @@ -20,7 +20,6 @@ impl Service { } } - pub fn with_store(store: Box) -> Self { Self { store, @@ -61,7 +60,11 @@ impl Storage for Service { let page_no = request.page_no; let namespace = request.namespace; trace!("find_frame(page_no={})", page_no); - if let Some(frame_no) = self.store.find_frame(&namespace, page_no).await { + if let Some(frame_no) = self + .store + .find_frame(&namespace, page_no, request.max_frame_no) + .await + { Ok(Response::new(rpc::FindFrameResponse { frame_no: Some(frame_no), })) diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index 9561fded93..98e32004d8 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -7,7 +7,7 @@ pub trait FrameStore: Send + Sync { #[allow(dead_code)] async fn insert_frames(&self, namespace: &str, frames: Vec) -> u64; async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option; - async fn find_frame(&self, namespace: &str, page_no: u32) -> Option; + async fn find_frame(&self, namespace: &str, page_no: u32, max_frame_no: u64) -> Option; async fn frame_page_no(&self, namespace: &str, frame_no: u64) -> Option; async fn frames_in_wal(&self, namespace: &str) -> u64; async fn destroy(&self, namespace: &str); diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 210d37d92b..71b07788a6 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -110,6 +110,7 @@ pub struct DurableWal { frames_cache: SieveCache>, write_cache: BTreeMap, lock_manager: Arc>, + max_frame_no: u64, } impl DurableWal { @@ -129,6 +130,7 @@ impl DurableWal { frames_cache: page_frames, write_cache: BTreeMap::new(), lock_manager, + max_frame_no: 0, } } @@ -141,7 +143,7 @@ impl DurableWal { let req = rpc::FindFrameRequest { namespace: self.namespace.clone(), page_no: page_no.get(), - max_frame_no: 0, + max_frame_no: self.max_frame_no, }; let mut binding = self.client.clone(); let resp = binding.find_frame(req).await.unwrap(); @@ -174,6 +176,7 @@ impl Wal for DurableWal { // - create a read lock // - save the current max_frame_no for this txn // + self.max_frame_no = self.db_size() as u64; Ok(true) } From 66fc5d85d4580bed0d2dd4b231d41e9651ac58be Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 8 Jun 2024 18:24:56 +0530 Subject: [PATCH 53/60] set max_frame_size in begin_tx --- libsql-storage/src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 71b07788a6..17245eed5e 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -176,7 +176,12 @@ impl Wal for DurableWal { // - create a read lock // - save the current max_frame_no for this txn // - self.max_frame_no = self.db_size() as u64; + let rt = tokio::runtime::Handle::current(); + let size = tokio::task::block_in_place(|| rt.block_on(self.frames_count())) + .try_into() + .unwrap(); + trace!("DurableWal::db_size() => {}", size); + self.max_frame_no = size; Ok(true) } From ed31b45dc2dcd72e87b6089f8e65f34cdd5da0ef Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 8 Jun 2024 18:29:32 +0530 Subject: [PATCH 54/60] send mx frame in insert req --- libsql-storage/proto/storage.proto | 1 + libsql-storage/src/generated/storage.rs | 2 ++ libsql-storage/src/lib.rs | 1 + 3 files changed, 4 insertions(+) diff --git a/libsql-storage/proto/storage.proto b/libsql-storage/proto/storage.proto index 17920e6f89..c2782a0a78 100644 --- a/libsql-storage/proto/storage.proto +++ b/libsql-storage/proto/storage.proto @@ -10,6 +10,7 @@ message Frame { message InsertFramesRequest { string namespace = 1; repeated Frame frames = 2; + uint64 max_frame_no = 3; } message InsertFramesResponse { diff --git a/libsql-storage/src/generated/storage.rs b/libsql-storage/src/generated/storage.rs index 908706e559..dd6e26d7a4 100644 --- a/libsql-storage/src/generated/storage.rs +++ b/libsql-storage/src/generated/storage.rs @@ -14,6 +14,8 @@ pub struct InsertFramesRequest { pub namespace: ::prost::alloc::string::String, #[prost(message, repeated, tag = "2")] pub frames: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "3")] + pub max_frame_no: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 17245eed5e..8c890463de 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -345,6 +345,7 @@ impl Wal for DurableWal { let req = rpc::InsertFramesRequest { namespace: self.namespace.clone(), frames: self.write_cache.values().cloned().collect(), + max_frame_no: self.max_frame_no, }; self.write_cache.clear(); let mut binding = self.client.clone(); From 9ad3d2d026fa3c0a63f4441d39c55c84d4ffc5f5 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 8 Jun 2024 18:58:08 +0530 Subject: [PATCH 55/60] Update proto to send `max_frame_no` while inserting frames We plan to allow multiple (non concurrent) writers. If a transaction's `max_frame_no` does not match the server's `max_frame_no`, then it has missed some new writes and we can abort the transaction --- libsql-storage/proto/storage.proto | 1 + libsql-storage/src/generated/storage.rs | 2 ++ libsql-storage/src/lib.rs | 1 + 3 files changed, 4 insertions(+) diff --git a/libsql-storage/proto/storage.proto b/libsql-storage/proto/storage.proto index 17920e6f89..c2782a0a78 100644 --- a/libsql-storage/proto/storage.proto +++ b/libsql-storage/proto/storage.proto @@ -10,6 +10,7 @@ message Frame { message InsertFramesRequest { string namespace = 1; repeated Frame frames = 2; + uint64 max_frame_no = 3; } message InsertFramesResponse { diff --git a/libsql-storage/src/generated/storage.rs b/libsql-storage/src/generated/storage.rs index 908706e559..dd6e26d7a4 100644 --- a/libsql-storage/src/generated/storage.rs +++ b/libsql-storage/src/generated/storage.rs @@ -14,6 +14,8 @@ pub struct InsertFramesRequest { pub namespace: ::prost::alloc::string::String, #[prost(message, repeated, tag = "2")] pub frames: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "3")] + pub max_frame_no: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/libsql-storage/src/lib.rs b/libsql-storage/src/lib.rs index 210d37d92b..d16e38dea7 100644 --- a/libsql-storage/src/lib.rs +++ b/libsql-storage/src/lib.rs @@ -337,6 +337,7 @@ impl Wal for DurableWal { let req = rpc::InsertFramesRequest { namespace: self.namespace.clone(), frames: self.write_cache.values().cloned().collect(), + max_frame_no: 0, }; self.write_cache.clear(); let mut binding = self.client.clone(); From d8ae21f2297d13347b8e057f29566652c3797687 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 8 Jun 2024 19:19:11 +0530 Subject: [PATCH 56/60] Updates storage trait's insert_frames definition - Change parameter type `FrameData` to `Frame` from rpc def, to avoid unnecessary copying - Take `max_frame_no` param --- libsql-storage-server/src/fdb_store.rs | 3 ++- libsql-storage-server/src/memory_store.rs | 8 +++++++- libsql-storage-server/src/redis_store.rs | 6 +++--- libsql-storage-server/src/store.rs | 3 ++- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index 7f753de61c..d853514795 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -5,6 +5,7 @@ use foundationdb::api::NetworkAutoStop; use foundationdb::tuple::pack; use foundationdb::tuple::unpack; use foundationdb::Transaction; +use libsql_storage::rpc::Frame; use tracing::error; pub struct FDBFrameStore { @@ -72,7 +73,7 @@ impl FrameStore for FDBFrameStore { frame_no } - async fn insert_frames(&self, namespace: &str, frames: Vec) -> u64 { + async fn insert_frames(&self, namespace: &str, max_frame_no: u64, frames: Vec) -> u64 { let max_frame_key = format!("{}/max_frame_no", namespace); let db = foundationdb::Database::default().unwrap(); let txn = db.create_trx().expect("unable to create transaction"); diff --git a/libsql-storage-server/src/memory_store.rs b/libsql-storage-server/src/memory_store.rs index 41cd427b14..23ff87c3e5 100644 --- a/libsql-storage-server/src/memory_store.rs +++ b/libsql-storage-server/src/memory_store.rs @@ -4,6 +4,7 @@ use std::sync::Mutex; use crate::store::FrameData; use crate::store::FrameStore; use async_trait::async_trait; +use libsql_storage::rpc::Frame; #[derive(Default)] pub(crate) struct InMemFrameStore { @@ -47,7 +48,12 @@ impl FrameStore for InMemFrameStore { frame_no } - async fn insert_frames(&self, _namespace: &str, _frames: Vec) -> u64 { + async fn insert_frames( + &self, + _namespace: &str, + _max_frame_no: u64, + _frames: Vec, + ) -> u64 { todo!() } diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index 14f32903dc..b8e99a2b8d 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -1,7 +1,7 @@ -use crate::store::FrameData; use crate::store::FrameStore; use async_trait::async_trait; use bytes::Bytes; +use libsql_storage::rpc::Frame; use redis::{Client, Commands, RedisResult}; use tracing::error; @@ -49,10 +49,10 @@ impl FrameStore for RedisFrameStore { max_frame_no } - async fn insert_frames(&self, namespace: &str, frames: Vec) -> u64 { + async fn insert_frames(&self, namespace: &str, _max_frame_no: u64, frames: Vec) -> u64 { let mut max_frame_no = 0; for f in frames { - max_frame_no = self.insert_frame(namespace, f.page_no, f.data).await; + max_frame_no = self.insert_frame(namespace, f.page_no, f.data.into()).await; } max_frame_no } diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index 9561fded93..2609b9c854 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -1,11 +1,12 @@ use async_trait::async_trait; use bytes::Bytes; +use libsql_storage::rpc::Frame; #[async_trait] pub trait FrameStore: Send + Sync { async fn insert_frame(&self, namespace: &str, page_no: u32, frame: bytes::Bytes) -> u64; #[allow(dead_code)] - async fn insert_frames(&self, namespace: &str, frames: Vec) -> u64; + async fn insert_frames(&self, namespace: &str, max_frame_no: u64, frames: Vec) -> u64; async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option; async fn find_frame(&self, namespace: &str, page_no: u32) -> Option; async fn frame_page_no(&self, namespace: &str, frame_no: u64) -> Option; From a821b661684dccbc9da2e62abeb9f6ef1a26c13e Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 8 Jun 2024 20:21:56 +0530 Subject: [PATCH 57/60] remove insert_frames --- libsql-storage-server/src/fdb_store.rs | 2 +- libsql-storage-server/src/memory_store.rs | 29 +++++++++++++++----- libsql-storage-server/src/redis_store.rs | 32 ++++++++++++++++++++--- libsql-storage-server/src/service.rs | 18 +++---------- 4 files changed, 56 insertions(+), 25 deletions(-) diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index d853514795..5c401a9fcd 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -87,7 +87,7 @@ impl FrameStore for FDBFrameStore { frame_no, FrameData { page_no: f.page_no, - data: f.data, + data: f.data.into(), }, ) .await; diff --git a/libsql-storage-server/src/memory_store.rs b/libsql-storage-server/src/memory_store.rs index 23ff87c3e5..2be53d3f2d 100644 --- a/libsql-storage-server/src/memory_store.rs +++ b/libsql-storage-server/src/memory_store.rs @@ -48,13 +48,28 @@ impl FrameStore for InMemFrameStore { frame_no } - async fn insert_frames( - &self, - _namespace: &str, - _max_frame_no: u64, - _frames: Vec, - ) -> u64 { - todo!() + async fn insert_frames(&self, _namespace: &str, _max_frame_no: u64, frames: Vec) -> u64 { + let mut inner = self.inner.lock().unwrap(); + for frame in frames { + let frame_no = inner.max_frame_no + 1; + inner.max_frame_no = frame_no; + let page_no = frame.page_no; + inner.frames.insert( + frame_no, + FrameData { + page_no, + data: frame.data.into(), + }, + ); + inner + .pages + .entry(page_no) + .or_insert_with(Vec::new) + .push(frame_no); + tracing::trace!("inserted for page {} frame {}", page_no, frame_no) + } + let count = inner.max_frame_no; + count } async fn read_frame(&self, _namespace: &str, frame_no: u64) -> Option { diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index b8e99a2b8d..1ba8ed23ec 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -51,9 +51,35 @@ impl FrameStore for RedisFrameStore { async fn insert_frames(&self, namespace: &str, _max_frame_no: u64, frames: Vec) -> u64 { let mut max_frame_no = 0; - for f in frames { - max_frame_no = self.insert_frame(namespace, f.page_no, f.data.into()).await; - } + let max_frame_key = format!("{}/max_frame_no", namespace); + let mut con = self.client.get_connection().unwrap(); + // max_frame_key might change if another client inserts a frame, so do + // all this in a transaction! + let (max_frame_no,): (u64,) = + redis::transaction(&mut con, &[&max_frame_key], |con, pipe| { + let result: RedisResult = con.get(max_frame_key.clone()); + if result.is_err() && !is_nil_response(result.as_ref().err().unwrap()) { + return Err(result.err().unwrap()); + } + max_frame_no = result.unwrap_or(0); + for frame in &frames { + let max_frame_no = max_frame_no + 1; + let frame_key = format!("f/{}/{}", namespace, max_frame_no); + let page_key = format!("p/{}/{}", namespace, frame.page_no); + + pipe.hset::>(frame_key.clone(), "f", frame.data.to_vec()) + .ignore() + .hset::(frame_key.clone(), "p", frame.page_no) + .ignore() + .set::(page_key, max_frame_no) + .ignore() + .set::(max_frame_key.clone(), max_frame_no) + .ignore() + .get(max_frame_key.clone()); + } + pipe.query(con) + }) + .unwrap(); max_frame_no } diff --git a/libsql-storage-server/src/service.rs b/libsql-storage-server/src/service.rs index dc3846cfb9..5077e04a96 100644 --- a/libsql-storage-server/src/service.rs +++ b/libsql-storage-server/src/service.rs @@ -34,21 +34,11 @@ impl Storage for Service { &self, request: Request, ) -> Result, Status> { - let mut num_frames = 0; let request = request.into_inner(); - let namespace = request.namespace; - for frame in request.frames.into_iter() { - trace!( - "inserted for page {} frame {}", - frame.page_no, - self.store - .insert_frame(&namespace, frame.page_no, frame.data.into()) - .await - ); - num_frames += 1; - self.db_size - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - } + let ns = request.namespace; + let frames = request.frames; + let max_frame_no = request.max_frame_no; + let num_frames = self.store.insert_frames(&ns, max_frame_no, frames).await as u32; Ok(Response::new(rpc::InsertFramesResponse { num_frames })) } From 097d4d02ec271a9339a1650972536505f3f465f5 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 8 Jun 2024 20:34:46 +0530 Subject: [PATCH 58/60] Use `insert_frames` method --- libsql-storage-server/src/fdb_store.rs | 2 +- libsql-storage-server/src/memory_store.rs | 29 ++++++++++++++++----- libsql-storage-server/src/redis_store.rs | 31 ++++++++++++++++++++--- libsql-storage-server/src/service.rs | 18 +++---------- 4 files changed, 55 insertions(+), 25 deletions(-) diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index d853514795..5c401a9fcd 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -87,7 +87,7 @@ impl FrameStore for FDBFrameStore { frame_no, FrameData { page_no: f.page_no, - data: f.data, + data: f.data.into(), }, ) .await; diff --git a/libsql-storage-server/src/memory_store.rs b/libsql-storage-server/src/memory_store.rs index 23ff87c3e5..2be53d3f2d 100644 --- a/libsql-storage-server/src/memory_store.rs +++ b/libsql-storage-server/src/memory_store.rs @@ -48,13 +48,28 @@ impl FrameStore for InMemFrameStore { frame_no } - async fn insert_frames( - &self, - _namespace: &str, - _max_frame_no: u64, - _frames: Vec, - ) -> u64 { - todo!() + async fn insert_frames(&self, _namespace: &str, _max_frame_no: u64, frames: Vec) -> u64 { + let mut inner = self.inner.lock().unwrap(); + for frame in frames { + let frame_no = inner.max_frame_no + 1; + inner.max_frame_no = frame_no; + let page_no = frame.page_no; + inner.frames.insert( + frame_no, + FrameData { + page_no, + data: frame.data.into(), + }, + ); + inner + .pages + .entry(page_no) + .or_insert_with(Vec::new) + .push(frame_no); + tracing::trace!("inserted for page {} frame {}", page_no, frame_no) + } + let count = inner.max_frame_no; + count } async fn read_frame(&self, _namespace: &str, frame_no: u64) -> Option { diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index b8e99a2b8d..7b719df963 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -51,9 +51,34 @@ impl FrameStore for RedisFrameStore { async fn insert_frames(&self, namespace: &str, _max_frame_no: u64, frames: Vec) -> u64 { let mut max_frame_no = 0; - for f in frames { - max_frame_no = self.insert_frame(namespace, f.page_no, f.data.into()).await; - } + let max_frame_key = format!("{}/max_frame_no", namespace); + let mut con = self.client.get_connection().unwrap(); + // max_frame_key might change if another client inserts a frame, so do + // all this in a transaction! + let (max_frame_no,): (u64,) = + redis::transaction(&mut con, &[&max_frame_key], |con, pipe| { + let result: RedisResult = con.get(max_frame_key.clone()); + if result.is_err() && !is_nil_response(result.as_ref().err().unwrap()) { + return Err(result.err().unwrap()); + } + max_frame_no = result.unwrap_or(0); + for frame in &frames { + let max_frame_no = max_frame_no + 1; + let frame_key = format!("f/{}/{}", namespace, max_frame_no); + let page_key = format!("p/{}/{}", namespace, frame.page_no); + + pipe.hset::>(frame_key.clone(), "f", frame.data.to_vec()) + .ignore() + .hset::(frame_key.clone(), "p", frame.page_no) + .ignore() + .set::(page_key, max_frame_no) + .ignore() + .set::(max_frame_key.clone(), max_frame_no) + .ignore(); + } + pipe.get(max_frame_key.clone()).query(con) + }) + .unwrap(); max_frame_no } diff --git a/libsql-storage-server/src/service.rs b/libsql-storage-server/src/service.rs index dc3846cfb9..5077e04a96 100644 --- a/libsql-storage-server/src/service.rs +++ b/libsql-storage-server/src/service.rs @@ -34,21 +34,11 @@ impl Storage for Service { &self, request: Request, ) -> Result, Status> { - let mut num_frames = 0; let request = request.into_inner(); - let namespace = request.namespace; - for frame in request.frames.into_iter() { - trace!( - "inserted for page {} frame {}", - frame.page_no, - self.store - .insert_frame(&namespace, frame.page_no, frame.data.into()) - .await - ); - num_frames += 1; - self.db_size - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - } + let ns = request.namespace; + let frames = request.frames; + let max_frame_no = request.max_frame_no; + let num_frames = self.store.insert_frames(&ns, max_frame_no, frames).await as u32; Ok(Response::new(rpc::InsertFramesResponse { num_frames })) } From 5b7e287583158fe3320ff812be70e6949a5da515 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 8 Jun 2024 20:36:35 +0530 Subject: [PATCH 59/60] cleanup: remove `insert_frame` method and impl --- libsql-storage-server/src/fdb_store.rs | 22 +--------------- libsql-storage-server/src/memory_store.rs | 19 -------------- libsql-storage-server/src/redis_store.rs | 31 ----------------------- libsql-storage-server/src/store.rs | 2 -- 4 files changed, 1 insertion(+), 73 deletions(-) diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index 5c401a9fcd..836edfa56a 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -53,27 +53,7 @@ impl FDBFrameStore { #[async_trait] impl FrameStore for FDBFrameStore { - async fn insert_frame(&self, namespace: &str, page_no: u32, frame: bytes::Bytes) -> u64 { - let max_frame_key = format!("{}/max_frame_no", namespace); - let db = foundationdb::Database::default().unwrap(); - let txn = db.create_trx().expect("unable to create transaction"); - let frame_no = self.get_max_frame_no(&txn, namespace).await + 1; - self.insert_with_tx( - namespace, - &txn, - frame_no, - FrameData { - page_no, - data: frame, - }, - ) - .await; - txn.set(&max_frame_key.as_bytes(), &pack(&(frame_no))); - txn.commit().await.expect("commit failed"); - frame_no - } - - async fn insert_frames(&self, namespace: &str, max_frame_no: u64, frames: Vec) -> u64 { + async fn insert_frames(&self, namespace: &str, _max_frame_no: u64, frames: Vec) -> u64 { let max_frame_key = format!("{}/max_frame_no", namespace); let db = foundationdb::Database::default().unwrap(); let txn = db.create_trx().expect("unable to create transaction"); diff --git a/libsql-storage-server/src/memory_store.rs b/libsql-storage-server/src/memory_store.rs index 2be53d3f2d..dd2e57f623 100644 --- a/libsql-storage-server/src/memory_store.rs +++ b/libsql-storage-server/src/memory_store.rs @@ -29,25 +29,6 @@ impl InMemFrameStore { #[async_trait] impl FrameStore for InMemFrameStore { // inserts a new frame for the page number and returns the new frame value - async fn insert_frame(&self, _namespace: &str, page_no: u32, frame: bytes::Bytes) -> u64 { - let mut inner = self.inner.lock().unwrap(); - let frame_no = inner.max_frame_no + 1; - inner.max_frame_no = frame_no; - inner.frames.insert( - frame_no, - FrameData { - page_no, - data: frame, - }, - ); - inner - .pages - .entry(page_no) - .or_insert_with(Vec::new) - .push(frame_no); - frame_no - } - async fn insert_frames(&self, _namespace: &str, _max_frame_no: u64, frames: Vec) -> u64 { let mut inner = self.inner.lock().unwrap(); for frame in frames { diff --git a/libsql-storage-server/src/redis_store.rs b/libsql-storage-server/src/redis_store.rs index 7b719df963..a580ca34b9 100644 --- a/libsql-storage-server/src/redis_store.rs +++ b/libsql-storage-server/src/redis_store.rs @@ -18,37 +18,6 @@ impl RedisFrameStore { #[async_trait] impl FrameStore for RedisFrameStore { - async fn insert_frame(&self, namespace: &str, page_no: u32, frame: bytes::Bytes) -> u64 { - let max_frame_key = format!("{}/max_frame_no", namespace); - - let mut con = self.client.get_connection().unwrap(); - // max_frame_key might change if another client inserts a frame, so do - // all this in a transaction! - let (max_frame_no,): (u64,) = - redis::transaction(&mut con, &[&max_frame_key], |con, pipe| { - let result: RedisResult = con.get(max_frame_key.clone()); - if result.is_err() && !is_nil_response(result.as_ref().err().unwrap()) { - return Err(result.err().unwrap()); - } - let max_frame_no = result.unwrap_or(0) + 1; - let frame_key = format!("f/{}/{}", namespace, max_frame_no); - let page_key = format!("p/{}/{}", namespace, page_no); - - pipe.hset::>(frame_key.clone(), "f", frame.to_vec()) - .ignore() - .hset::(frame_key.clone(), "p", page_no) - .ignore() - .set::(page_key, max_frame_no) - .ignore() - .set::(max_frame_key.clone(), max_frame_no) - .ignore() - .get(max_frame_key.clone()) - .query(con) - }) - .unwrap(); - max_frame_no - } - async fn insert_frames(&self, namespace: &str, _max_frame_no: u64, frames: Vec) -> u64 { let mut max_frame_no = 0; let max_frame_key = format!("{}/max_frame_no", namespace); diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index 2609b9c854..e887f0461b 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -4,8 +4,6 @@ use libsql_storage::rpc::Frame; #[async_trait] pub trait FrameStore: Send + Sync { - async fn insert_frame(&self, namespace: &str, page_no: u32, frame: bytes::Bytes) -> u64; - #[allow(dead_code)] async fn insert_frames(&self, namespace: &str, max_frame_no: u64, frames: Vec) -> u64; async fn read_frame(&self, namespace: &str, frame_no: u64) -> Option; async fn find_frame(&self, namespace: &str, page_no: u32) -> Option; From 66854f4bac747284914e35ed4f718b821bc09613 Mon Sep 17 00:00:00 2001 From: Avinash Sajjanshetty Date: Sat, 8 Jun 2024 20:42:39 +0530 Subject: [PATCH 60/60] Remove `FrameData`, use `Frame` from proto instead --- libsql-storage-server/src/fdb_store.rs | 14 ++------------ libsql-storage-server/src/memory_store.rs | 13 +++---------- libsql-storage-server/src/store.rs | 7 ------- 3 files changed, 5 insertions(+), 29 deletions(-) diff --git a/libsql-storage-server/src/fdb_store.rs b/libsql-storage-server/src/fdb_store.rs index 836edfa56a..8352a808f8 100644 --- a/libsql-storage-server/src/fdb_store.rs +++ b/libsql-storage-server/src/fdb_store.rs @@ -1,4 +1,3 @@ -use crate::store::FrameData; use crate::store::FrameStore; use async_trait::async_trait; use foundationdb::api::NetworkAutoStop; @@ -39,7 +38,7 @@ impl FDBFrameStore { namespace: &str, txn: &Transaction, frame_no: u64, - frame: FrameData, + frame: Frame, ) { let frame_data_key = format!("{}/f/{}/f", namespace, frame_no); let frame_page_key = format!("{}/f/{}/p", namespace, frame_no); @@ -61,16 +60,7 @@ impl FrameStore for FDBFrameStore { for f in frames { frame_no += 1; - self.insert_with_tx( - namespace, - &txn, - frame_no, - FrameData { - page_no: f.page_no, - data: f.data.into(), - }, - ) - .await; + self.insert_with_tx(namespace, &txn, frame_no, f).await; } txn.set(&max_frame_key.as_bytes(), &pack(&(frame_no))); txn.commit().await.expect("commit failed"); diff --git a/libsql-storage-server/src/memory_store.rs b/libsql-storage-server/src/memory_store.rs index dd2e57f623..38de84aa60 100644 --- a/libsql-storage-server/src/memory_store.rs +++ b/libsql-storage-server/src/memory_store.rs @@ -1,7 +1,6 @@ use std::collections::BTreeMap; use std::sync::Mutex; -use crate::store::FrameData; use crate::store::FrameStore; use async_trait::async_trait; use libsql_storage::rpc::Frame; @@ -14,7 +13,7 @@ pub(crate) struct InMemFrameStore { #[derive(Default)] struct InMemInternal { // contains a frame data, key is the frame number - frames: BTreeMap, + frames: BTreeMap, // pages map contains the page number as a key and the list of frames for the page as a value pages: BTreeMap>, max_frame_no: u64, @@ -35,13 +34,7 @@ impl FrameStore for InMemFrameStore { let frame_no = inner.max_frame_no + 1; inner.max_frame_no = frame_no; let page_no = frame.page_no; - inner.frames.insert( - frame_no, - FrameData { - page_no, - data: frame.data.into(), - }, - ); + inner.frames.insert(frame_no, frame); inner .pages .entry(page_no) @@ -59,7 +52,7 @@ impl FrameStore for InMemFrameStore { .unwrap() .frames .get(&frame_no) - .map(|frame| frame.data.clone()) + .map(|frame| frame.data.clone().into()) } // given a page number, return the maximum frame for the page diff --git a/libsql-storage-server/src/store.rs b/libsql-storage-server/src/store.rs index e887f0461b..f30cd11ee9 100644 --- a/libsql-storage-server/src/store.rs +++ b/libsql-storage-server/src/store.rs @@ -1,5 +1,4 @@ use async_trait::async_trait; -use bytes::Bytes; use libsql_storage::rpc::Frame; #[async_trait] @@ -11,9 +10,3 @@ pub trait FrameStore: Send + Sync { async fn frames_in_wal(&self, namespace: &str) -> u64; async fn destroy(&self, namespace: &str); } - -#[derive(Default)] -pub struct FrameData { - pub(crate) page_no: u32, - pub(crate) data: Bytes, -}