From de57dc9031f56945c8901e94ad417b72e201036e Mon Sep 17 00:00:00 2001 From: Michael Ye Date: Sun, 4 Dec 2022 18:25:15 -0500 Subject: [PATCH 01/10] style: reformat fuse client --- .clang-format | 3 +- src/service/fuse/fuse_client.cpp | 221 +++-- src/service/fuse/fuse_client_context.hpp | 1057 +++++++++++----------- 3 files changed, 625 insertions(+), 656 deletions(-) diff --git a/.clang-format b/.clang-format index 7e4a8374..c7e91cf5 100644 --- a/.clang-format +++ b/.clang-format @@ -12,7 +12,7 @@ AllowShortLoopsOnASingleLine: true AllowShortFunctionsOnASingleLine: All AlwaysBreakTemplateDeclarations: true AlwaysBreakBeforeMultilineStrings: false -BreakBeforeBinaryOperators: true +BreakBeforeBinaryOperators: true BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: false BinPackParameters: true @@ -53,4 +53,3 @@ ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] SpaceBeforeParens: Never DisableFormat: false ... - diff --git a/src/service/fuse/fuse_client.cpp b/src/service/fuse/fuse_client.cpp index b0dc45d8..ef270ce0 100644 --- a/src/service/fuse/fuse_client.cpp +++ b/src/service/fuse/fuse_client.cpp @@ -1,24 +1,22 @@ #define FUSE_USE_VERSION 31 -#include -#include -#include -#include -#include -#include -#include #include "fuse_client_context.hpp" #include #include #include - +#include +#include +#include +#include +#include +#include #include -#define FUSE_CLIENT_DEV_ID (0xCA7CADE) +#define FUSE_CLIENT_DEV_ID (0xCA7CADE) /** * fuse_client mount the cascade service to file system. This allows users to access cascade data with normal POSIX * filesystem API. - * + * * The data in cascade is organized this way: * ///// * "mount-point" is where the cascade data is mounted. @@ -36,34 +34,34 @@ using FuseClientContextType = FuseClientContext(p) #define FCC_REQ(req) FCC(fuse_req_userdata(req)) -static void fs_init(void* userdata, struct fuse_conn_info *conn) { - dbg_default_trace("entering {}.",__func__); - if (derecho::hasCustomizedConfKey(CONF_LAYOUT_JSON_LAYOUT)) { - FCC(userdata)->initialize(json::parse(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT))); - } else if (derecho::hasCustomizedConfKey(CONF_LAYOUT_JSON_LAYOUT_FILE)){ +static void fs_init(void* userdata, struct fuse_conn_info* conn) { + dbg_default_trace("entering {}.", __func__); + if(derecho::hasCustomizedConfKey(CONF_LAYOUT_JSON_LAYOUT)) { + FCC(userdata)->initialize(json::parse(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT))); + } else if(derecho::hasCustomizedConfKey(CONF_LAYOUT_JSON_LAYOUT_FILE)) { nlohmann::json layout_array; std::ifstream json_file(derecho::getAbsoluteFilePath(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT_FILE))); - if (!json_file) { - dbg_default_error("Cannot load json configuration from file: {}", derecho::getAbsoluteFilePath(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT_FILE))); + if(!json_file) { + dbg_default_error("Cannot load json configuration from file: {}", derecho::getAbsoluteFilePath(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT_FILE))); throw derecho::derecho_exception("Cannot load json configuration from file."); } json_file >> layout_array; FCC(userdata)->initialize(layout_array); } - dbg_default_trace("leaving {}.",__func__); + dbg_default_trace("leaving {}.", __func__); } static void fs_destroy(void* userdata) { - dbg_default_trace("entering {}.",__func__); - dbg_default_trace("leaving {}.",__func__); + dbg_default_trace("entering {}.", __func__); + dbg_default_trace("leaving {}.", __func__); } static void fs_lookup(fuse_req_t req, fuse_ino_t parent, const char* name) { - dbg_default_trace("entering {}.",__func__); + dbg_default_trace("entering {}.", __func__); struct fuse_entry_param e; // TODO: make this more efficient by implement a dedicated call in FCC. auto name_to_ino = FCC_REQ(req)->get_dir_entries(parent); - if (name_to_ino.find(name) == name_to_ino.end()) { + if(name_to_ino.find(name) == name_to_ino.end()) { fuse_reply_err(req, ENOENT); } else { // TODO: change timeout settings. @@ -75,79 +73,79 @@ static void fs_lookup(fuse_req_t req, fuse_ino_t parent, const char* name) { fuse_reply_entry(req, &e); } - dbg_default_trace("leaving {}.",__func__); + dbg_default_trace("leaving {}.", __func__); } static void fs_getattr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { - dbg_default_trace("entering {}.",__func__); + dbg_default_trace("entering {}.", __func__); struct stat stbuf; - (void) fi; + (void)fi; std::memset(&stbuf, 0, sizeof(stbuf)); stbuf.st_ino = ino; FCC_REQ(req)->fill_stbuf_by_ino(stbuf); fuse_reply_attr(req, &stbuf, 10000.0); - dbg_default_trace("leaving {}.",__func__); + dbg_default_trace("leaving {}.", __func__); } // borrowed from libfuse \a hello_ll.c struct dirbuf { - char *p; + char* p; size_t size; }; -static void dirbuf_add(fuse_req_t req, struct dirbuf *b, const char *name, fuse_ino_t ino) { +static void dirbuf_add(fuse_req_t req, struct dirbuf* b, const char* name, fuse_ino_t ino) { struct stat stbuf; size_t oldsize = b->size; b->size += fuse_add_direntry(req, nullptr, 0, name, nullptr, 0); - b->p = (char*)realloc(b->p,b->size); - std::memset(&stbuf,0,sizeof(stbuf)); + b->p = (char*)realloc(b->p, b->size); + std::memset(&stbuf, 0, sizeof(stbuf)); stbuf.st_ino = ino; FCC_REQ(req)->fill_stbuf_by_ino(stbuf); - fuse_add_direntry(req, b->p+oldsize, b->size-oldsize, name, &stbuf, b->size); + fuse_add_direntry(req, b->p + oldsize, b->size - oldsize, name, &stbuf, b->size); } #define min(x, y) ((x) < (y) ? (x) : (y)) static void fs_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, struct fuse_file_info* fi) { - dbg_default_trace("entering {}.",__func__); + dbg_default_trace("entering {}.", __func__); struct dirbuf b; std::memset(&b, 0, sizeof(b)); dirbuf_add(req, &b, ".", 1); dirbuf_add(req, &b, "..", 1); - for(auto kv:FCC_REQ(req)->get_dir_entries(ino)) { + for(auto kv : FCC_REQ(req)->get_dir_entries(ino)) { dirbuf_add(req, &b, kv.first.c_str(), kv.second); } - if (static_cast(off) < b.size) { - fuse_reply_buf(req, b.p + off, min(b.size - off, size)); + if(static_cast(off) < b.size) { + fuse_reply_buf(req, b.p + off, min(b.size - off, size)); } else { fuse_reply_buf(req, NULL, 0); } free(b.p); - dbg_default_trace("leaving {}.",__func__); + dbg_default_trace("leaving {}.", __func__); } static void fs_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { - dbg_default_trace("entering {}.",__func__); + dbg_default_trace("entering {}.", __func__); int err; - if ((fi->flags & O_ACCMODE) != O_RDONLY) { + if((fi->flags & O_ACCMODE) != O_RDONLY) { fuse_reply_err(req, EACCES); - } else if ((err = FCC_REQ(req)->open_file(ino, fi)) != 0) { + } else if((err = FCC_REQ(req)->open_file(ino, fi)) != 0) { fuse_reply_err(req, err); } else { - dbg_default_debug("fi({:x})->fh={:x}",reinterpret_cast(fi),fi->fh); + dbg_default_debug("fi({:x})->fh={:x}", reinterpret_cast(fi), fi->fh); fuse_reply_open(req, fi); } - dbg_default_trace("leaving {}.",__func__); + dbg_default_trace("leaving {}.", __func__); } static void fs_read(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, struct fuse_file_info* fi) { dbg_default_trace("entering {}.", __func__); FileBytes* pfb = reinterpret_cast(fi->fh); - if (static_cast(off) < pfb->size) { - fuse_reply_buf(req, reinterpret_cast(pfb->bytes+off), min(pfb->size - off, size)); + if(static_cast(off) < pfb->size) { + fuse_reply_buf(req, reinterpret_cast(pfb->bytes + off), min(pfb->size - off, size)); } else { fuse_reply_buf(req, nullptr, 0); } @@ -155,38 +153,37 @@ static void fs_read(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, stru } static void fs_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { - dbg_default_trace("entering {}.",__func__); + dbg_default_trace("entering {}.", __func__); FCC_REQ(req)->close_file(ino, fi); - fuse_reply_err(req,0); - dbg_default_trace("leaving {}.",__func__); + fuse_reply_err(req, 0); + dbg_default_trace("leaving {}.", __func__); } static const struct fuse_lowlevel_ops fs_ops = { - .init = fs_init, - .destroy = fs_destroy, - .lookup = fs_lookup, - .forget = NULL, - .getattr = fs_getattr, - .setattr = NULL, - .readlink = NULL, - .mknod = NULL, - .mkdir = NULL, - .unlink = NULL, - .rmdir = NULL, - .symlink = NULL, - .rename = NULL, - .link = NULL, - .open = fs_open, - .read = fs_read, - .write = NULL, - .flush = NULL, - .release = fs_release, - .fsync = NULL, - .opendir = NULL, - .readdir = fs_readdir, + .init = fs_init, + .destroy = fs_destroy, + .lookup = fs_lookup, + .forget = NULL, + .getattr = fs_getattr, + .setattr = NULL, + .readlink = NULL, + .mknod = NULL, + .mkdir = NULL, + .unlink = NULL, + .rmdir = NULL, + .symlink = NULL, + .rename = NULL, + .link = NULL, + .open = fs_open, + .read = fs_read, + .write = NULL, + .flush = NULL, + .release = fs_release, + .fsync = NULL, + .opendir = NULL, + .readdir = fs_readdir, }; - /** * According to our experiment as well as recorded in * [this](https://www.cs.hmc.edu/~geoff/classes/hmc.cs135.201109/homework/fuse/fuse_doc.html) document as following: @@ -201,39 +198,37 @@ static const struct fuse_lowlevel_ops fs_ops = { void prepare_derecho_conf_file() { char cwd[4096]; #pragma GCC diagnostic ignored "-Wunused-result" - getcwd(cwd,4096); + getcwd(cwd, 4096); #pragma GCC diagnostic pop - sprintf(cwd+strlen(cwd), "/derecho.cfg"); + sprintf(cwd + strlen(cwd), "/derecho.cfg"); setenv("DERECHO_CONF_FILE", cwd, false); dbg_default_debug("Using derecho config file:{}.", getenv("DERECHO_CONF_FILE")); } void log_current_dir(bool foreground) { - char buff[FILENAME_MAX]; //create string buffer to hold path - auto working = GetCurrentDir( buff, FILENAME_MAX ); - if(working){ - std::string current_working_dir(buff); - - current_working_dir = (foreground)? current_working_dir + " (running in foreground)": current_working_dir + " (running in background)"; - auto derecho_conf_file = getenv("DERECHO_CONF_FILE"); - - std::ofstream out("/home/yy354/fuse_log.txt"); - out << current_working_dir; - out << " derecho conf file: [" <" << std::endl; fuse_cmdline_help(); fuse_lowlevel_help(); ret = 0; throw 1; - } else if (opts.show_version) { + } else if(opts.show_version) { std::cout << "FUSE library version " << fuse_pkgversion() << std::endl; fuse_lowlevel_version(); ret = 0; throw 1; } - if (opts.mountpoint == nullptr) { + if(opts.mountpoint == nullptr) { std::cout << "usage: " << argv[0] << " [options] " << std::endl; ret = 1; throw 1; } - fuse_daemonize(opts.foreground); + fuse_daemonize(opts.foreground); // start session FuseClientContextType fcc; se = fuse_session_new(&args, &fs_ops, sizeof(fs_ops), &fcc); - if (se == nullptr) { + if(se == nullptr) { throw 1; } - if (fuse_set_signal_handlers(se) != 0) { + if(fuse_set_signal_handlers(se) != 0) { throw 2; } - if (fuse_session_mount(se, opts.mountpoint) != 0) { + if(fuse_session_mount(se, opts.mountpoint) != 0) { throw 3; } - //log_current_dir(opts.foreground); + // log_current_dir(opts.foreground); - /* Block until ctrl+c or fuserount -u */ - if (opts.singlethread) { + /* Block until ctrl+c or fuserount -u */ + if(opts.singlethread) { ret = fuse_session_loop(se); } else { ret = fuse_session_loop_mt(se, opts.clone_fd); } - + fuse_session_unmount(se); - } - catch (int& ex) { + } catch(int& ex) { switch(ex) { - case 3: - fuse_remove_signal_handlers(se); - case 2: - fuse_session_destroy(se); - case 1: - free(opts.mountpoint); - fuse_opt_free_args(&args); + case 3: + fuse_remove_signal_handlers(se); + case 2: + fuse_session_destroy(se); + case 1: + free(opts.mountpoint); + fuse_opt_free_args(&args); } } diff --git a/src/service/fuse/fuse_client_context.hpp b/src/service/fuse/fuse_client_context.hpp index 96f9aecd..0328468b 100644 --- a/src/service/fuse/fuse_client_context.hpp +++ b/src/service/fuse/fuse_client_context.hpp @@ -1,30 +1,30 @@ #pragma once -#include +#include "cascade/object_pool_metadata.hpp" #include -#include -#include -#include -#include #include +#include #include +#include +#include +#include +#include #include -#include +#include #include -#include "cascade/object_pool_metadata.hpp" #include #define GetCurrentDir getcwd namespace derecho { -namespace cascade{ +namespace cascade { using json = nlohmann::json; -#define FUSE_CLIENT_DEV_ID (0xCA7CADE) -#define FUSE_CLIENT_BLK_SIZE (4096) +#define FUSE_CLIENT_DEV_ID (0xCA7CADE) +#define FUSE_CLIENT_BLK_SIZE (4096) -#define META_FILE_NAME ".cascade" +#define META_FILE_NAME ".cascade" -#define TO_FOREVER (std::numeric_limits::max()) +#define TO_FOREVER (std::numeric_limits::max()) typedef enum { SITE = 0, @@ -43,15 +43,15 @@ class FileBytes { public: size_t size; uint8_t* bytes; - FileBytes():size(0),bytes(nullptr){} - FileBytes(size_t s):size(s) { + FileBytes() : size(0), bytes(nullptr) {} + FileBytes(size_t s) : size(s) { bytes = nullptr; - if (s > 0) { - bytes = (uint8_t*)malloc(s); + if(s > 0) { + bytes = (uint8_t*)malloc(s); } } virtual ~FileBytes() { - if (bytes){ + if(bytes) { free(bytes); } } @@ -71,10 +71,10 @@ class FuseClientINode { * get directory entries. This is the default implementation. * Override it as required. */ - virtual std::map get_dir_entries() { - std::map ret_map; - for (auto& child: this->children) { - ret_map.emplace(child->display_name,reinterpret_cast(child.get())); + virtual std::map get_dir_entries() { + std::map ret_map; + for(auto& child : this->children) { + ret_map.emplace(child->display_name, reinterpret_cast(child.get())); } return ret_map; } @@ -84,51 +84,49 @@ class FuseClientINode { } virtual uint64_t read_file(FileBytes* fb) { - (void) fb; + (void)fb; return 0; } - virtual void initialize(){ + virtual void initialize() { } - + // Helper function for get_dir_entries() and read_file() - void check_update(){ - struct timespec now; + void check_update() { + struct timespec now; clock_gettime(CLOCK_REALTIME, &now); - if (now.tv_sec > (last_update_sec + update_interval)){ + if(now.tv_sec > (last_update_sec + update_interval)) { clock_gettime(CLOCK_REALTIME, &now); - if (now.tv_sec > (last_update_sec + update_interval)){ - last_update_sec = now.tv_sec; + if(now.tv_sec > (last_update_sec + update_interval)) { + last_update_sec = now.tv_sec; update_contents(); } } } private: - // Helper functions for check_update() + // Helper functions for check_update() virtual void update_contents() { return; } }; - template -struct TypeName { static const char *name; }; +struct TypeName { static const char* name; }; template -const char *TypeName::name = "unknown"; +const char* TypeName::name = "unknown"; template <> -const char *TypeName::name = "VolatileCascadeStoreWithStringKey"; +const char* TypeName::name = "VolatileCascadeStoreWithStringKey"; template <> -const char *TypeName::name = "PersistentCascadeStoreWithStringKey"; +const char* TypeName::name = "PersistentCascadeStoreWithStringKey"; template <> -const char *TypeName::name = "TriggerCascadeNoStoreWithStringKey"; +const char* TypeName::name = "TriggerCascadeNoStoreWithStringKey"; -#define CONF_LAYOUT "layout" - +#define CONF_LAYOUT "layout" template class SubgroupINode; @@ -163,25 +161,24 @@ template class CascadeTypeINode : public FuseClientINode { public: CascadeTypeINode() { - this->type = INodeType::CASCADE_TYPE; - this->display_name = std::string(TypeName::name); - this->parent = FUSE_ROOT_ID; + this->type = INodeType::CASCADE_TYPE; + this->display_name = std::string(TypeName::name); + this->parent = FUSE_ROOT_ID; } /** initialize */ void initialize(const json& group_layout, ServiceClientAPI& capi) { this->display_name = group_layout["type_alias"]; - uint32_t sidx=0; - for (auto subgroup_it:group_layout[CONF_LAYOUT]) { - - children.emplace_back(std::make_unique>(sidx,reinterpret_cast(this))); + uint32_t sidx = 0; + for(auto subgroup_it : group_layout[CONF_LAYOUT]) { + children.emplace_back(std::make_unique>(sidx, reinterpret_cast(this))); size_t num_shards = subgroup_it[MIN_NODES_BY_SHARD].size(); - for (uint32_t shidx = 0; shidx < num_shards; shidx ++) { + for(uint32_t shidx = 0; shidx < num_shards; shidx++) { this->children[sidx]->children.emplace_back( - std::make_unique>( - shidx,reinterpret_cast(this->children[sidx].get()),capi)); + std::make_unique>( + shidx, reinterpret_cast(this->children[sidx].get()), capi)); } - sidx ++; + sidx++; } } }; @@ -189,10 +186,9 @@ class CascadeTypeINode : public FuseClientINode { class RootMetaINode : public FuseClientINode { ServiceClientAPI& capi; std::string contents; - + public: - RootMetaINode (ServiceClientAPI& _capi) : - capi(_capi) { + RootMetaINode(ServiceClientAPI& _capi) : capi(_capi) { this->update_interval = 2; this->last_update_sec = 0; this->type = INodeType::META; @@ -200,24 +196,24 @@ class RootMetaINode : public FuseClientINode { } virtual uint64_t get_file_size() override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - check_update(); + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + check_update(); return contents.size(); } private: - virtual void update_contents () override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); + virtual void update_contents() override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); auto members = capi.get_members(); contents = "number of nodes in cascade service: " + std::to_string(members.size()) + ".\nnode IDs: "; - for (auto& nid : members) { + for(auto& nid : members) { contents += std::to_string(nid) + ","; } contents += "\n"; - + auto objectpools = capi.list_object_pools(true); contents += "number of objectpool in cascade service: " + std::to_string(objectpools.size()) + ".\nObjectpool paths: "; - for (auto& objectpool_path : objectpools) { + for(auto& objectpool_path : objectpools) { contents += objectpool_path + ","; } contents += "\n"; @@ -236,7 +232,7 @@ class SubgroupINode : public FuseClientINode { public: const uint32_t subgroup_index; - SubgroupINode (uint32_t sidx, fuse_ino_t pino) : subgroup_index(sidx) { + SubgroupINode(uint32_t sidx, fuse_ino_t pino) : subgroup_index(sidx) { this->type = INodeType::SUBGROUP; this->display_name = "subgroup-" + std::to_string(sidx); this->parent = pino; @@ -250,41 +246,40 @@ class ShardINode : public FuseClientINode { ServiceClientAPI& capi; std::map key_to_ino; - ShardINode (uint32_t shidx, fuse_ino_t pino, ServiceClientAPI& _capi) : - shard_index (shidx), capi(_capi) { + ShardINode(uint32_t shidx, fuse_ino_t pino, ServiceClientAPI& _capi) : shard_index(shidx), capi(_capi) { this->type = INodeType::SHARD; this->display_name = "shard-" + std::to_string(shidx); this->parent = pino; SubgroupINode* p_subgroup_inode = reinterpret_cast*>(pino); - this->children.emplace_back(std::make_unique>(shidx,p_subgroup_inode->subgroup_index,capi)); + this->children.emplace_back(std::make_unique>(shidx, p_subgroup_inode->subgroup_index, capi)); } // TODO: rethinking about the consistency - virtual std::map get_dir_entries() override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - //std::map ret_map; + virtual std::map get_dir_entries() override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + // std::map ret_map; /** we always retrieve the key list for a shard inode because the data is highly dynamic */ uint32_t subgroup_index = reinterpret_cast*>(this->parent)->subgroup_index; - auto result = capi.template list_keys(CURRENT_VERSION, true, subgroup_index, this->shard_index); - for (auto& reply_future:result.get()) { + auto result = capi.template list_keys(CURRENT_VERSION, true, subgroup_index, this->shard_index); + for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); std::unique_lock wlck(this->children_mutex); - for (auto& key: reply) { - if (key_to_ino.find(key) == key_to_ino.end()) { - this->children.emplace_back(std::make_unique>(key,reinterpret_cast(this),capi)); - // To solve the issue of '/' in display name , which will cause: reading directory '.': input/output error - // TODO: if there are better replacement of /, instead of - - key = std::string("key-") + key; - std::replace( key.begin(), key.end(), '/', '-'); + for(auto& key : reply) { + if(key_to_ino.find(key) == key_to_ino.end()) { + this->children.emplace_back(std::make_unique>(key, reinterpret_cast(this), capi)); + // To solve the issue of '/' in display name , which will cause: reading directory '.': input/output error + // TODO: if there are better replacement of /, instead of - + key = std::string("key-") + key; + std::replace(key.begin(), key.end(), '/', '-'); key_to_ino[key] = reinterpret_cast(this->children.back().get()); } } } - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); - return FuseClientINode::get_dir_entries(); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return FuseClientINode::get_dir_entries(); } }; - + template <> class ShardINode : public FuseClientINode { public: @@ -292,19 +287,18 @@ class ShardINode : public FuseClientINode { ServiceClientAPI& capi; std::map key_to_ino; - ShardINode (uint32_t shidx, fuse_ino_t pino, ServiceClientAPI& _capi) : - shard_index (shidx), capi(_capi) { + ShardINode(uint32_t shidx, fuse_ino_t pino, ServiceClientAPI& _capi) : shard_index(shidx), capi(_capi) { this->type = INodeType::SHARD; this->display_name = "shard-" + std::to_string(shidx); this->parent = pino; SubgroupINode* p_subgroup_inode = reinterpret_cast*>(pino); - this->children.emplace_back(std::make_unique>(shidx,p_subgroup_inode->subgroup_index,capi)); + this->children.emplace_back(std::make_unique>(shidx, p_subgroup_inode->subgroup_index, capi)); } - virtual std::map get_dir_entries() override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - std::map ret_map; - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); + virtual std::map get_dir_entries() override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + std::map ret_map; + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); return ret_map; } }; @@ -320,52 +314,52 @@ class ShardMetaINode : public FuseClientINode { /** * update the metadata. need write lock. */ - virtual void update_contents () override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - auto members = capi.template get_shard_members(subgroup_index,shard_index); + virtual void update_contents() override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + auto members = capi.template get_shard_members(subgroup_index, shard_index); contents = "number of nodes shard: " + std::to_string(members.size()) + ".\nnode IDs: "; - for (auto& nid : members) { + for(auto& nid : members) { contents += std::to_string(nid) + ","; } contents += "\n"; - auto policy = capi.template get_member_selection_policy(subgroup_index,shard_index); + auto policy = capi.template get_member_selection_policy(subgroup_index, shard_index); contents += "member selection policy:"; switch(std::get<0>(policy)) { - case FirstMember: - contents += "FirstMember\n"; - break; - case LastMember: - contents += "LastMember\n"; - break; - case Random: - contents += "Random\n"; - break; - case FixedRandom: - contents += "FixedRandom("; - contents += std::to_string(std::get<1>(policy)); - contents += ")\n"; - break; - case RoundRobin: - contents += "RoundRobin\n"; - break; - case UserSpecified: - contents += "UserSpecified("; - contents += std::to_string(std::get<1>(policy)); - contents += ")\n"; - break; - default: - contents += "Unknown\n"; - break; + case FirstMember: + contents += "FirstMember\n"; + break; + case LastMember: + contents += "LastMember\n"; + break; + case Random: + contents += "Random\n"; + break; + case FixedRandom: + contents += "FixedRandom("; + contents += std::to_string(std::get<1>(policy)); + contents += ")\n"; + break; + case RoundRobin: + contents += "RoundRobin\n"; + break; + case UserSpecified: + contents += "UserSpecified("; + contents += std::to_string(std::get<1>(policy)); + contents += ")\n"; + break; + default: + contents += "Unknown\n"; + break; } - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } + public: - ShardMetaINode (const uint32_t _shard_index, const uint32_t _subgroup_index, ServiceClientAPI& _capi) : - shard_index(_shard_index), - subgroup_index(_subgroup_index), - capi (_capi) { + ShardMetaINode(const uint32_t _shard_index, const uint32_t _subgroup_index, ServiceClientAPI& _capi) : shard_index(_shard_index), + subgroup_index(_subgroup_index), + capi(_capi) { this->update_interval = 2; - this->last_update_sec = 0; + this->last_update_sec = 0; this->type = INodeType::META; this->display_name = META_FILE_NAME; } @@ -376,55 +370,52 @@ class ShardMetaINode : public FuseClientINode { } virtual uint64_t read_file(FileBytes* file_bytes) override { - check_update(); - file_bytes->size = contents.size(); - file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); - return 0; + check_update(); + file_bytes->size = contents.size(); + file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); + return 0; } }; - template class KeyINode : public FuseClientINode { public: typename CascadeType::KeyType key; std::unique_ptr file_bytes; uint64_t file_size; - persistent::version_t version; - uint64_t timestamp_us; - persistent::version_t previous_version; - persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. - ServiceClientAPI& capi; - - KeyINode(typename CascadeType::KeyType& k, fuse_ino_t pino, ServiceClientAPI& _capi) : - key(k), - file_bytes(std::make_unique()), - capi(_capi){ + persistent::version_t version; + uint64_t timestamp_us; + persistent::version_t previous_version; + persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. + ServiceClientAPI& capi; + + KeyINode(typename CascadeType::KeyType& k, fuse_ino_t pino, ServiceClientAPI& _capi) : key(k), + file_bytes(std::make_unique()), + capi(_capi) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); - this->update_interval = 2; + this->update_interval = 2; this->last_update_sec = 0; this->type = INodeType::KEY; - if constexpr (std::is_same, char*>::value || - std::is_same, std::string>::value) { + if constexpr(std::is_same, char*>::value || std::is_same, std::string>::value) { this->display_name = std::string("key-") + k; - } else if constexpr (std::is_arithmetic>::value) { + } else if constexpr(std::is_arithmetic>::value) { this->display_name = std::string("key-") + std::to_string(k); } else { // KeyType is required to implement to_string() for types other than type string/arithmetic. this->display_name = std::string("key-") + key.to_string(); } // '/' in display name, will cause: reading directory '.': input/output error - std::replace( this->display_name.begin(), this->display_name.end(), '/', '\\'); - this->parent = pino; + std::replace(this->display_name.begin(), this->display_name.end(), '/', '\\'); + this->parent = pino; dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } virtual uint64_t read_file(FileBytes* _file_bytes) override { dbg_default_trace("[{}]entering {}.", gettid(), __func__); check_update(); - _file_bytes->size = this->file_bytes.get()->size; - _file_bytes->bytes = static_cast(malloc(this->file_bytes.get()->size)); - memcpy(_file_bytes->bytes,this->file_bytes.get()->bytes, this->file_bytes.get()->size); + _file_bytes->size = this->file_bytes.get()->size; + _file_bytes->bytes = static_cast(malloc(this->file_bytes.get()->size)); + memcpy(_file_bytes->bytes, this->file_bytes.get()->bytes, this->file_bytes.get()->size); dbg_default_trace("[{}]leaving {}.", gettid(), __func__); return 0; } @@ -434,148 +425,147 @@ class KeyINode : public FuseClientINode { return this->file_bytes.get()->size; } - KeyINode(KeyINode&& fci){ + KeyINode(KeyINode&& fci) { this->type = std::move(fci.type); this->display_name = std::move(fci.display_name); this->parent = std::move(fci.parent); - this->key = std::move(fci.key ); + this->key = std::move(fci.key); this->capi = std::move(fci.capi); } virtual ~KeyINode() { dbg_default_info("[{}] entering {}.", gettid(), __func__); - file_bytes.reset(nullptr); + file_bytes.reset(nullptr); dbg_default_info("[{}] leaving {}.", gettid(), __func__); } + private: - virtual void update_contents () override{ - ShardINode *pino_shard = reinterpret_cast*>(this->parent); - SubgroupINode *pino_subgroup = reinterpret_cast*>(pino_shard->parent); + virtual void update_contents() override { + ShardINode* pino_shard = reinterpret_cast*>(this->parent); + SubgroupINode* pino_subgroup = reinterpret_cast*>(pino_shard->parent); auto result = capi.template get( - key,CURRENT_VERSION,true,pino_subgroup->subgroup_index,pino_shard->shard_index); - Blob blob; - for(auto& reply_future:result.get()){ - auto reply = reply_future.second.get(); - dbg_default_trace("------ KEY INODE reply {}.", reply); - if(std::is_base_of::value){ - ObjectWithStringKey *access_ptr = reinterpret_cast(&reply); - this->version = access_ptr->version; - this->timestamp_us = access_ptr->timestamp_us; - this->previous_version = access_ptr->previous_version; - this->previous_version_by_key = access_ptr->previous_version_by_key; - blob = access_ptr->blob; - }else if(std::is_base_of::value){ - ObjectWithUInt64Key *access_ptr = reinterpret_cast(&reply); - this->version = access_ptr->version; - this->timestamp_us = access_ptr->timestamp_us; - this->previous_version = access_ptr->previous_version; - this->previous_version_by_key = access_ptr->previous_version_by_key; - blob = access_ptr->blob; - } - else{ - this->file_bytes.get()->size = mutils::bytes_size(reply); - this->file_bytes.get()->bytes = static_cast(malloc(this->file_bytes.get()->size)); - reply.to_bytes(this->file_bytes.get()->bytes); - return; - } - file_bytes.get()->size = blob.size; - file_bytes.get()->bytes = static_cast(malloc(file_bytes.get()->size)); - memcpy(file_bytes.get()->bytes, blob.bytes, file_bytes.get()->size); - return; + key, CURRENT_VERSION, true, pino_subgroup->subgroup_index, pino_shard->shard_index); + Blob blob; + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + dbg_default_trace("------ KEY INODE reply {}.", reply); + if(std::is_base_of::value) { + ObjectWithStringKey* access_ptr = reinterpret_cast(&reply); + this->version = access_ptr->version; + this->timestamp_us = access_ptr->timestamp_us; + this->previous_version = access_ptr->previous_version; + this->previous_version_by_key = access_ptr->previous_version_by_key; + blob = access_ptr->blob; + } else if(std::is_base_of::value) { + ObjectWithUInt64Key* access_ptr = reinterpret_cast(&reply); + this->version = access_ptr->version; + this->timestamp_us = access_ptr->timestamp_us; + this->previous_version = access_ptr->previous_version; + this->previous_version_by_key = access_ptr->previous_version_by_key; + blob = access_ptr->blob; + } else { + this->file_bytes.get()->size = mutils::bytes_size(reply); + this->file_bytes.get()->bytes = static_cast(malloc(this->file_bytes.get()->size)); + reply.to_bytes(this->file_bytes.get()->bytes); + return; + } + file_bytes.get()->size = blob.size; + file_bytes.get()->bytes = static_cast(malloc(file_bytes.get()->size)); + memcpy(file_bytes.get()->bytes, blob.bytes, file_bytes.get()->size); + return; } } }; - class ObjectPoolMetaINode : public FuseClientINode { private: std::string cur_pathname; /** objp_collection contains the all the objp with the same cur_pathname prefix - * i.e. if cur_path name is "/a", + * i.e. if cur_path name is "/a", * object pools "/a/b1", "/a/b2" share this level of ObjectPoolPathINode*/ std::vector objp_collection; - bool is_object_pool; - uint32_t subgroup_type_index; - uint32_t subgroup_index; + bool is_object_pool; + uint32_t subgroup_type_index; + uint32_t subgroup_index; sharding_policy_t sharding_policy; - bool deleted; + bool deleted; ServiceClientAPI& capi; std::string contents; - virtual void update_contents () override{ + virtual void update_contents() override { this->contents = "Current Directory Pathname: "; - this->contents += (cur_pathname=="") ? "objectPoolRoot" : cur_pathname; - this->contents += "\n"; - this->objp_collection.clear(); - this->contents += "contains the below object pools in its subdirs:\n"; - std::string objp_contents; - // Check the objectPools under cur_pathname directory - size_t cur_len = cur_pathname.length(); - for (std::string pathname : capi.list_object_pools(true)) { - if(cur_pathname.length() == 0 || (pathname.length() > cur_len - 1 && (pathname.substr(0,cur_len) == cur_pathname) )){ - if(pathname.length() == cur_len){ - this->objp_collection.emplace_back(pathname); - this->contents += " " + pathname + ",\n"; - this->is_object_pool = true; - this->objectPool_contents( objp_contents); - }else if(pathname[cur_len] == '/'){ - this->objp_collection.emplace_back(pathname); - this->contents += " " + pathname + ",\n"; - } - } - } - this->contents += objp_contents; - } - + this->contents += (cur_pathname == "") ? "objectPoolRoot" : cur_pathname; + this->contents += "\n"; + this->objp_collection.clear(); + this->contents += "contains the below object pools in its subdirs:\n"; + std::string objp_contents; + // Check the objectPools under cur_pathname directory + size_t cur_len = cur_pathname.length(); + for(std::string pathname : capi.list_object_pools(true)) { + if(cur_pathname.length() == 0 || (pathname.length() > cur_len - 1 && (pathname.substr(0, cur_len) == cur_pathname))) { + if(pathname.length() == cur_len) { + this->objp_collection.emplace_back(pathname); + this->contents += " " + pathname + ",\n"; + this->is_object_pool = true; + this->objectPool_contents(objp_contents); + } else if(pathname[cur_len] == '/') { + this->objp_collection.emplace_back(pathname); + this->contents += " " + pathname + ",\n"; + } + } + } + this->contents += objp_contents; + } + /** * Only fill the object pool contents when cur_pathname is an object pool */ - void objectPool_contents ( std::string& objp_contents) { + void objectPool_contents(std::string& objp_contents) { auto op_metadata = capi.find_object_pool(this->cur_pathname); - objp_contents += "Current Object Pool Pathname: "; - objp_contents += cur_pathname + "\n"; + objp_contents += "Current Object Pool Pathname: "; + objp_contents += cur_pathname + "\n"; this->deleted = op_metadata.deleted; - objp_contents += "- is deleted: "; - objp_contents += this->deleted? "true": "false"; - objp_contents += "\n"; + objp_contents += "- is deleted: "; + objp_contents += this->deleted ? "true" : "false"; + objp_contents += "\n"; this->subgroup_type_index = op_metadata.subgroup_type_index; - objp_contents += "- subgroup type index: "; - objp_contents += std::to_string(this->subgroup_type_index) + "\n"; + objp_contents += "- subgroup type index: "; + objp_contents += std::to_string(this->subgroup_type_index) + "\n"; this->subgroup_index = op_metadata.subgroup_index; - objp_contents += "- subgroup index: "; - objp_contents += std::to_string(this->subgroup_index) + "\n"; + objp_contents += "- subgroup index: "; + objp_contents += std::to_string(this->subgroup_index) + "\n"; auto policy = op_metadata.sharding_policy; objp_contents += "- sharding policy: "; switch(policy) { - case HASH: - objp_contents += "Hashing\n"; - break; - case RANGE: - objp_contents += "Range\n"; - break; - default: - objp_contents += "Unknown\n"; - break; + case HASH: + objp_contents += "Hashing\n"; + break; + case RANGE: + objp_contents += "Range\n"; + break; + default: + objp_contents += "Unknown\n"; + break; } } + public: - ObjectPoolMetaINode (const std::string _cur_pathname, ServiceClientAPI& _capi) : - cur_pathname(_cur_pathname), - capi(_capi) { + ObjectPoolMetaINode(const std::string _cur_pathname, ServiceClientAPI& _capi) : cur_pathname(_cur_pathname), + capi(_capi) { this->update_interval = 2; this->last_update_sec = 0; this->type = INodeType::META; this->display_name = META_FILE_NAME; } - void add_objp(std::string new_objp_pathname){ - for(auto &pathname : objp_collection){ - if(pathname == new_objp_pathname){ - return; - } - } - objp_collection.emplace_back(new_objp_pathname); - } + void add_objp(std::string new_objp_pathname) { + for(auto& pathname : objp_collection) { + if(pathname == new_objp_pathname) { + return; + } + } + objp_collection.emplace_back(new_objp_pathname); + } virtual uint64_t get_file_size() override { check_update(); @@ -583,13 +573,12 @@ class ObjectPoolMetaINode : public FuseClientINode { } virtual uint64_t read_file(FileBytes* file_bytes) override { - check_update(); - file_bytes->size = contents.size(); - file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); - return 0; + check_update(); + file_bytes->size = contents.size(); + file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); + return 0; } }; - class ObjectPoolPathINode : public FuseClientINode { public: @@ -599,227 +588,221 @@ class ObjectPoolPathINode : public FuseClientINode { std::set key_children; std::set objp_children; - - ObjectPoolPathINode (fuse_ino_t pino, ServiceClientAPI& _capi): - capi(_capi){ - this->update_interval = 10; - this->last_update_sec = 0; + ObjectPoolPathINode(fuse_ino_t pino, ServiceClientAPI& _capi) : capi(_capi) { + this->update_interval = 10; + this->last_update_sec = 0; this->type = INodeType::OBJECTPOOL_PATH; this->parent = pino; - this->is_object_pool = false; - this->cur_pathname = ""; - this->children.emplace_back(std::make_unique(cur_pathname, capi)); + this->is_object_pool = false; + this->cur_pathname = ""; + this->children.emplace_back(std::make_unique(cur_pathname, capi)); } - ObjectPoolPathINode (std::string _cur_pathname,fuse_ino_t pino, ServiceClientAPI& _capi) : - capi(_capi), - cur_pathname(_cur_pathname){ + ObjectPoolPathINode(std::string _cur_pathname, fuse_ino_t pino, ServiceClientAPI& _capi) : capi(_capi), + cur_pathname(_cur_pathname) { this->update_interval = 10; this->last_update_sec = 10; this->type = INodeType::OBJECTPOOL_PATH; - std::size_t pos = cur_pathname.find_last_of('/'); + std::size_t pos = cur_pathname.find_last_of('/'); this->display_name = cur_pathname.substr(pos + 1); this->parent = pino; - this->is_object_pool = false; - //this->objp_collection.emplace_back(objp_pathname); - this->children.emplace_back(std::make_unique(cur_pathname, capi)); + this->is_object_pool = false; + // this->objp_collection.emplace_back(objp_pathname); + this->children.emplace_back(std::make_unique(cur_pathname, capi)); } - /** Helper function: + /** Helper function: * @object_pool_pathname: the object pool pathname to parse * @return: next level pathname * e.g. if cur_pathname is "/a", for object_pool_pathname "/a/b/c" this function returns "/a/b" */ - std::string get_next_level_pathname(std::string &object_pool_pathname){ - size_t cur_len = cur_pathname.length(); - std::string remain_pathname = object_pool_pathname.substr(cur_len); + std::string get_next_level_pathname(std::string& object_pool_pathname) { + size_t cur_len = cur_pathname.length(); + std::string remain_pathname = object_pool_pathname.substr(cur_len); auto start_pos = remain_pathname.find("/"); - if (start_pos == std::string::npos){ + if(start_pos == std::string::npos) { return ""; } - // get the next level of directory pathname. + // get the next level of directory pathname. auto end_pos = remain_pathname.substr(start_pos + 1).find("/"); std::string next_level_pathname; - if (end_pos == std::string::npos){ - next_level_pathname = cur_pathname + remain_pathname.substr(start_pos ); - }else{ - next_level_pathname = cur_pathname + remain_pathname.substr(start_pos , end_pos + 1); + if(end_pos == std::string::npos) { + next_level_pathname = cur_pathname + remain_pathname.substr(start_pos); + } else { + next_level_pathname = cur_pathname + remain_pathname.substr(start_pos, end_pos + 1); } - return next_level_pathname; - } + return next_level_pathname; + } - /** Construct the next level objectPoolINodes, starting from remain pathname - *e.g./a/b/c should have three layers: /a, /a/b, /a/b/c - *if this->cur_pathname is "/a", whose child INode's cur_pathname is "/a/b", whose child INode "/a/b/c" + /** Construct the next level objectPoolINodes, starting from remain pathname + *e.g./a/b/c should have three layers: /a, /a/b, /a/b/c + *if this->cur_pathname is "/a", whose child INode's cur_pathname is "/a/b", whose child INode "/a/b/c" */ - void construct_nextlevel_objectpool_path(std::string &object_pool_pathname){ + void construct_nextlevel_objectpool_path(std::string& object_pool_pathname) { // check path_name - std::string next_level_pathname = get_next_level_pathname(object_pool_pathname); - if(next_level_pathname.length() == 0){ - return; - } - // Check if this objectPoolPathInode with the same pathname exists - // case 1: If this level pathname directory exists - for(auto& inode : this->children){ - if (inode->type == INodeType::OBJECTPOOL_PATH){ - if(static_cast(inode.get())->cur_pathname == next_level_pathname){ - return; - } - } + std::string next_level_pathname = get_next_level_pathname(object_pool_pathname); + if(next_level_pathname.length() == 0) { + return; } - // case2: If this level ObjectPoolPathInode doesn't exists, create one + // Check if this objectPoolPathInode with the same pathname exists + // case 1: If this level pathname directory exists + for(auto& inode : this->children) { + if(inode->type == INodeType::OBJECTPOOL_PATH) { + if(static_cast(inode.get())->cur_pathname == next_level_pathname) { + return; + } + } + } + // case2: If this level ObjectPoolPathInode doesn't exists, create one // std::unique_lock wlck(this->children_mutex); - this->children.emplace_back(std::make_unique(next_level_pathname,reinterpret_cast(this),capi)); - objp_children.insert(cur_pathname); + this->children.emplace_back(std::make_unique(next_level_pathname, reinterpret_cast(this), capi)); + objp_children.insert(cur_pathname); } - - void update_objpINodes(){ + void update_objpINodes() { size_t cur_len = cur_pathname.length(); - std::vector reply = capi.list_object_pools(true); - std::vector valid_subdirs; + std::vector reply = capi.list_object_pools(true); + std::vector valid_subdirs; // Check if need to add new ObjectPoolPathINode to children inodes - for (std::string &object_pool : reply) { - if(object_pool.length() < cur_len){ - continue; - } - if(object_pool == cur_pathname){ - this->is_object_pool = true; - continue; - } - bool is_subdir = cur_pathname.length() == 0 || (object_pool.substr(0,cur_len) == cur_pathname && object_pool[cur_len] == '/') ; - if(!is_subdir){ - continue; - } - std::string next_level_pathname(get_next_level_pathname(object_pool)); - valid_subdirs.emplace_back(next_level_pathname); - if( objp_children.find(next_level_pathname) == objp_children.end() ){ - construct_nextlevel_objectpool_path(object_pool); - } + for(std::string& object_pool : reply) { + if(object_pool.length() < cur_len) { + continue; + } + if(object_pool == cur_pathname) { + this->is_object_pool = true; + continue; + } + bool is_subdir = cur_pathname.length() == 0 || (object_pool.substr(0, cur_len) == cur_pathname && object_pool[cur_len] == '/'); + if(!is_subdir) { + continue; + } + std::string next_level_pathname(get_next_level_pathname(object_pool)); + valid_subdirs.emplace_back(next_level_pathname); + if(objp_children.find(next_level_pathname) == objp_children.end()) { + construct_nextlevel_objectpool_path(object_pool); + } + } + // Check if need to remove existing ObjectPoolPathINode from children inodes + if(std::find(reply.begin(), reply.end(), this->cur_pathname) == reply.end()) { + this->is_object_pool = false; } - // Check if need to remove existing ObjectPoolPathINode from children inodes - if(std::find(reply.begin(), reply.end(), this->cur_pathname) == reply.end()){ - this->is_object_pool = false; - } auto it = this->children.begin(); - while(it != this->children.end()){ - std::string name = cur_pathname + "/" + (*it)->display_name; - if ((*it)->type == INodeType::OBJECTPOOL_PATH && (std::find(valid_subdirs.begin(),valid_subdirs.end(), name ) == valid_subdirs.end())){ - objp_children.erase(objp_children.find(name)); - it = this->children.erase(it); - }else{ - ++it; - } - } - } - - void update_keyINodes(){ - // case1. if object pool of cur_pathname donesn't exists anymore, remove all the keyINode from children - if(!this->is_object_pool){ - auto it = this->children.begin(); - while(it != this->children.end()){ - if ((*it)->type == INodeType::KEY){ - it = this->children.erase(it); - }else{ - ++it; - } - } - return; - } - // case2. if cur_pathname is a valid object pool, refetch keys in this object pool - persistent::version_t version = CURRENT_VERSION; - std::vector>>> future_result = capi.list_keys(version, true, cur_pathname); - std::vector reply = capi.wait_list_keys(future_result); - // Check if need to add new keyINode to children inodes - for (auto& key: reply) { - if(key_children.find(key) == key_children.end() ){ - this->children.emplace_back(std::make_unique(key,reinterpret_cast(this),capi)); - key_children.insert(key); - } - } - // Check if need to remove existing keyINode from children inodes - auto it = this->children.begin(); - while(it != this->children.end()){ - std::string name = cur_pathname + "/" + (*it)->display_name; - if ((*it)->type == INodeType::KEY && (std::find(reply.begin(),reply.end(), name ) == reply.end())){ - key_children.erase(key_children.find(name)); - it = this->children.erase(it); - - }else{ - ++it; - } - } - } - - - virtual std::map get_dir_entries() override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - check_update(); - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); - return FuseClientINode::get_dir_entries(); + while(it != this->children.end()) { + std::string name = cur_pathname + "/" + (*it)->display_name; + if((*it)->type == INodeType::OBJECTPOOL_PATH && (std::find(valid_subdirs.begin(), valid_subdirs.end(), name) == valid_subdirs.end())) { + objp_children.erase(objp_children.find(name)); + it = this->children.erase(it); + } else { + ++it; + } + } } + + void update_keyINodes() { + // case1. if object pool of cur_pathname donesn't exists anymore, remove all the keyINode from children + if(!this->is_object_pool) { + auto it = this->children.begin(); + while(it != this->children.end()) { + if((*it)->type == INodeType::KEY) { + it = this->children.erase(it); + } else { + ++it; + } + } + return; + } + // case2. if cur_pathname is a valid object pool, refetch keys in this object pool + persistent::version_t version = CURRENT_VERSION; + std::vector>>> future_result = capi.list_keys(version, true, cur_pathname); + std::vector reply = capi.wait_list_keys(future_result); + // Check if need to add new keyINode to children inodes + for(auto& key : reply) { + if(key_children.find(key) == key_children.end()) { + this->children.emplace_back(std::make_unique(key, reinterpret_cast(this), capi)); + key_children.insert(key); + } + } + // Check if need to remove existing keyINode from children inodes + auto it = this->children.begin(); + while(it != this->children.end()) { + std::string name = cur_pathname + "/" + (*it)->display_name; + if((*it)->type == INodeType::KEY && (std::find(reply.begin(), reply.end(), name) == reply.end())) { + key_children.erase(key_children.find(name)); + it = this->children.erase(it); + + } else { + ++it; + } + } + } + + virtual std::map get_dir_entries() override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + check_update(); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return FuseClientINode::get_dir_entries(); + } + private: - virtual void update_contents () override{ - update_objpINodes(); - update_keyINodes(); - } + virtual void update_contents() override { + update_objpINodes(); + update_keyINodes(); + } }; - + class ObjectPoolRootINode : public ObjectPoolPathINode { public: - ObjectPoolRootINode (ServiceClientAPI& _capi, fuse_ino_t pino=FUSE_ROOT_ID) : - ObjectPoolPathINode(pino, _capi) { + ObjectPoolRootINode(ServiceClientAPI& _capi, fuse_ino_t pino = FUSE_ROOT_ID) : ObjectPoolPathINode(pino, _capi) { this->display_name = "ObjectPools"; this->parent = pino; } - /** this function constructs the whole object pool directories at the beginning - * for all the object pools stored in metadata service - */ - virtual std::map get_dir_entries() override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - check_update(); - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); - return FuseClientINode::get_dir_entries(); + /** this function constructs the whole object pool directories at the beginning + * for all the object pools stored in metadata service + */ + virtual std::map get_dir_entries() override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + check_update(); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return FuseClientINode::get_dir_entries(); } + private: - virtual void update_contents () override{ - update_objpINodes(); - update_keyINodes(); - } + virtual void update_contents() override { + update_objpINodes(); + update_keyINodes(); + } }; - -class ObjectPoolKeyINode : public FuseClientINode{ +class ObjectPoolKeyINode : public FuseClientINode { public: std::string key; - std::unique_ptr file_bytes; - persistent::version_t version; - uint64_t timestamp_us; - persistent::version_t previous_version; - persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. + std::unique_ptr file_bytes; + persistent::version_t version; + uint64_t timestamp_us; + persistent::version_t previous_version; + persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. ServiceClientAPI& capi; - ObjectPoolKeyINode(std::string k, fuse_ino_t pino, ServiceClientAPI& _capi) : - key(k), - file_bytes(std::make_unique()), - capi(_capi){ + ObjectPoolKeyINode(std::string k, fuse_ino_t pino, ServiceClientAPI& _capi) : key(k), + file_bytes(std::make_unique()), + capi(_capi) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); this->update_interval = 2; this->last_update_sec = 0; std::size_t pos = k.find_last_of('/'); this->display_name = k.substr(pos + 1); this->parent = pino; - this->type = INodeType::KEY; + this->type = INodeType::KEY; dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } virtual uint64_t read_file(FileBytes* _file_bytes) override { - dbg_default_debug("-- READ FILE of key:[{}], [{}]entering {}.",this->key ,gettid(), __func__); - check_update(); - _file_bytes->size = this->file_bytes.get()->size; - _file_bytes->bytes = static_cast(malloc(this->file_bytes.get()->size)); - memcpy(_file_bytes->bytes,this->file_bytes.get()->bytes, this->file_bytes.get()->size); + dbg_default_debug("-- READ FILE of key:[{}], [{}]entering {}.", this->key, gettid(), __func__); + check_update(); + _file_bytes->size = this->file_bytes.get()->size; + _file_bytes->bytes = static_cast(malloc(this->file_bytes.get()->size)); + memcpy(_file_bytes->bytes, this->file_bytes.get()->bytes, this->file_bytes.get()->size); dbg_default_debug("[{}]leaving {}.", gettid(), __func__); return 0; } @@ -827,33 +810,34 @@ class ObjectPoolKeyINode : public FuseClientINode{ virtual uint64_t get_file_size() override { dbg_default_debug("----GET FILE SIZE key is [{}].", this->key); check_update(); - return this->file_bytes.get()->size; + return this->file_bytes.get()->size; } virtual ~ObjectPoolKeyINode() { dbg_default_info("[{}] entering {}.", gettid(), __func__); - file_bytes.reset(nullptr); + file_bytes.reset(nullptr); dbg_default_info("[{}] leaving {}.", gettid(), __func__); } + private: - virtual void update_contents () override{ - dbg_default_debug("\n \n ----OBJP keyInode key is:[{}] - update content [{}] entering {}.", this->key ,gettid(), __func__); + virtual void update_contents() override { + dbg_default_debug("\n \n ----OBJP keyInode key is:[{}] - update content [{}] entering {}.", this->key, gettid(), __func__); Blob blob; - auto result = capi.get(key,CURRENT_VERSION,true); - for (auto& reply_future:result.get()) { + auto result = capi.get(key, CURRENT_VERSION, true); + for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); - ObjectWithStringKey *access_ptr = reinterpret_cast(&reply); - this->version = access_ptr->version; - this->timestamp_us = access_ptr->timestamp_us; - this->previous_version = access_ptr->previous_version; - this->previous_version_by_key = access_ptr->previous_version_by_key; - blob = access_ptr->blob; - this->file_bytes.get()->size = blob.size; - this->file_bytes.get()->bytes = static_cast(malloc(blob.size)); - memcpy(this->file_bytes.get()->bytes, blob.bytes, blob.size); - return; + ObjectWithStringKey* access_ptr = reinterpret_cast(&reply); + this->version = access_ptr->version; + this->timestamp_us = access_ptr->timestamp_us; + this->previous_version = access_ptr->previous_version; + this->previous_version_by_key = access_ptr->previous_version_by_key; + blob = access_ptr->blob; + this->file_bytes.get()->size = blob.size; + this->file_bytes.get()->bytes = static_cast(malloc(blob.size)); + memcpy(this->file_bytes.get()->bytes, blob.bytes, blob.size); + return; } - dbg_default_trace("\n \n ----OBJP keyInode update content [{}] leaving {}.", gettid(), __func__); + dbg_default_trace("\n \n ----OBJP keyInode update content [{}] leaving {}.", gettid(), __func__); } }; @@ -861,32 +845,29 @@ class MetadataServiceRootINode : public FuseClientINode { ServiceClientAPI& capi; public: - MetadataServiceRootINode(ServiceClientAPI& _capi) : - capi(_capi) { + MetadataServiceRootINode(ServiceClientAPI& _capi) : capi(_capi) { this->type = INodeType::METADATA_SERVICE; this->display_name = "MetadataService"; this->parent = FUSE_ROOT_ID; } void initialize() { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); + dbg_default_trace("[{}]entering {}.", gettid(), __func__); children.emplace_back(std::make_unique(capi, reinterpret_cast(this))); this->children.back().get()->initialize(); children.emplace_back(std::make_unique(capi, reinterpret_cast(this))); this->children.back().get()->initialize(); } - }; - /** - * TODO: add DLL FUSE support - */ +/** + * TODO: add DLL FUSE support + */ class DataPathLogicRootINode : public FuseClientINode { ServiceClientAPI& capi; public: - DataPathLogicRootINode (ServiceClientAPI& _capi, fuse_ino_t pino) : - capi(_capi) { + DataPathLogicRootINode(ServiceClientAPI& _capi, fuse_ino_t pino) : capi(_capi) { this->type = INodeType::DATAPATH_LOGIC; this->display_name = "DataPathLogic"; this->parent = FUSE_ROOT_ID; @@ -895,28 +876,26 @@ class DataPathLogicRootINode : public FuseClientINode { /** initialize */ void initialize() { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); + dbg_default_trace("[{}]entering {}.", gettid(), __func__); } }; - - /** - * TODO: add DLL FUSE support - */ +/** + * TODO: add DLL FUSE support + */ template class DLLINode : public FuseClientINode { public: - std::string file_name; // DLL id? + std::string file_name; // DLL id? ServiceClientAPI& capi; - DLLINode(std::string& _filename, fuse_ino_t pino, ServiceClientAPI& _capi) : - file_name(_filename), capi(_capi) { + DLLINode(std::string& _filename, fuse_ino_t pino, ServiceClientAPI& _capi) : file_name(_filename), capi(_capi) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); this->type = INodeType::DLL; this->display_name = std::string("dllfile") + _filename; this->parent = pino; dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } - + virtual uint64_t read_file(FileBytes* file_bytes) override { dbg_default_trace("[{}]entering {}.", gettid(), __func__); dbg_default_trace("[{}]leaving {}.", gettid(), __func__); @@ -930,11 +909,11 @@ class DLLINode : public FuseClientINode { return fsize; } - DLLINode(DLLINode&& fci){ + DLLINode(DLLINode&& fci) { this->type = std::move(fci.type); this->display_name = std::move(fci.display_name); this->parent = std::move(fci.parent); - this->file_name = std::move(fci.file_name ); + this->file_name = std::move(fci.file_name); this->capi = std::move(fci.capi); } @@ -944,14 +923,15 @@ class DLLINode : public FuseClientINode { } }; - /** * The fuse filesystem context for fuse_client. This context will be used as 'userdata' on starting a fuse session. */ template class FuseClientContext { - template using _CascadeTypeINode = CascadeTypeINode; + template + using _CascadeTypeINode = CascadeTypeINode; + private: /** initialization flag */ std::atomic is_initialized; @@ -963,7 +943,7 @@ class FuseClientContext { ServiceClientAPI& capi; /** The inodes are stored in \a inodes. */ - mutils::KindMap<_CascadeTypeINode,CascadeTypes...> inodes; + mutils::KindMap<_CascadeTypeINode, CascadeTypes...> inodes; /** Metadata */ RootMetaINode metadata_inode; @@ -976,12 +956,12 @@ class FuseClientContext { /** fill inodes */ void populate_inodes(const json& group_layout) { - if (!group_layout.is_array()) { + if(!group_layout.is_array()) { dbg_default_error("JSON group layout is invalid (array expected): {}.", group_layout.get()); throw std::runtime_error("JSON group layout is invalid."); } // populate from the second layout, since the first one is for metadata service - do_populate_inodes(group_layout,1); + do_populate_inodes(group_layout, 1); } template @@ -991,41 +971,40 @@ class FuseClientContext { template void do_populate_inodes(const json& group_layout, int type_idx) { this->do_populate_inodes(group_layout, type_idx); - this->do_populate_inodes(group_layout, type_idx+1); + this->do_populate_inodes(group_layout, type_idx + 1); } public: - FuseClientContext() : - capi(ServiceClientAPI::get_service_client()), - metadata_inode(capi), - objectpool_inode(capi), - admin_metadata_inode(capi){} + FuseClientContext() : capi(ServiceClientAPI::get_service_client()), + metadata_inode(capi), + objectpool_inode(capi), + admin_metadata_inode(capi) {} /** initialize */ void initialize(const json& group_layout) { dbg_default_trace("[{}]entering {} .", gettid(), __func__); populate_inodes(group_layout); this->admin_metadata_inode.initialize(); - clock_gettime(CLOCK_REALTIME,&this->init_timestamp); + clock_gettime(CLOCK_REALTIME, &this->init_timestamp); this->is_initialized.store(true); dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } /** read directory entries by ino */ - std::map get_dir_entries(fuse_ino_t ino) { + std::map get_dir_entries(fuse_ino_t ino) { dbg_default_trace("[{}]entering {} with ino ={:x}.", gettid(), __func__, ino); std::map ret_map; - if (ino == FUSE_ROOT_ID) { + if(ino == FUSE_ROOT_ID) { this->inodes.for_each( - [&ret_map](auto k,auto& v){ - // CAUTION: only works up to 64bit virtual address CPU architectures. - ret_map.emplace(v.display_name,reinterpret_cast(&v)); - }); - ret_map.emplace(metadata_inode.display_name,reinterpret_cast(&this->metadata_inode)); - ret_map.emplace(objectpool_inode.display_name,reinterpret_cast(&this->objectpool_inode)); - ret_map.emplace(admin_metadata_inode.display_name,reinterpret_cast(&this->admin_metadata_inode)); + [&ret_map](auto k, auto& v) { + // CAUTION: only works up to 64bit virtual address CPU architectures. + ret_map.emplace(v.display_name, reinterpret_cast(&v)); + }); + ret_map.emplace(metadata_inode.display_name, reinterpret_cast(&this->metadata_inode)); + ret_map.emplace(objectpool_inode.display_name, reinterpret_cast(&this->objectpool_inode)); + ret_map.emplace(admin_metadata_inode.display_name, reinterpret_cast(&this->admin_metadata_inode)); } else { FuseClientINode* pfci = reinterpret_cast(ino); - ret_map = pfci->get_dir_entries(); // RVO + ret_map = pfci->get_dir_entries(); // RVO } dbg_default_trace(" [{}]leaving {}.", gettid(), __func__); return ret_map; @@ -1037,8 +1016,8 @@ class FuseClientContext { * */ double fill_stbuf_by_ino(struct stat& stbuf) { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - double timeout_sec = 1.0; //TO_FOREVER; + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + double timeout_sec = 1.0; // TO_FOREVER; // 1 - common attributes stbuf.st_dev = FUSE_CLIENT_DEV_ID; stbuf.st_nlink = 1; @@ -1048,76 +1027,75 @@ class FuseClientContext { stbuf.st_mtim = this->init_timestamp; stbuf.st_ctim = this->init_timestamp; // 2 - special attributes - if (stbuf.st_ino == FUSE_ROOT_ID) { + if(stbuf.st_ino == FUSE_ROOT_ID) { stbuf.st_mode = S_IFDIR | 0755; stbuf.st_size = FUSE_CLIENT_BLK_SIZE; stbuf.st_blocks = 1; stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; } else { FuseClientINode* pfci = reinterpret_cast(stbuf.st_ino); - switch (pfci->type) { - case INodeType::SITE: - // TODO: - break; - case INodeType::CASCADE_TYPE: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::METADATA_SERVICE: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::SUBGROUP: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::SHARD: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::KEY: - stbuf.st_mode = S_IFREG| 0444; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::META: - stbuf.st_mode = S_IFREG| 0444; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::OBJECTPOOL_PATH: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::DATAPATH_LOGIC: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::DLL: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - default: - ; + switch(pfci->type) { + case INodeType::SITE: + // TODO: + break; + case INodeType::CASCADE_TYPE: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::METADATA_SERVICE: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::SUBGROUP: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::SHARD: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::KEY: + stbuf.st_mode = S_IFREG | 0444; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::META: + stbuf.st_mode = S_IFREG | 0444; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::OBJECTPOOL_PATH: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::DATAPATH_LOGIC: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::DLL: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + default:; } } - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); return timeout_sec; } @@ -1127,17 +1105,16 @@ class FuseClientContext { * @param fi file structure shared among processes opening this file. * @return error code. 0 for success. */ - int open_file(fuse_ino_t ino, struct fuse_file_info *fi) { + int open_file(fuse_ino_t ino, struct fuse_file_info* fi) { dbg_default_trace("[{}]entering {} with ino={:x}.", gettid(), __func__, ino); FuseClientINode* pfci = reinterpret_cast(ino); - if (pfci->type != INodeType::KEY && - pfci->type != INodeType::META) { + if(pfci->type != INodeType::KEY && pfci->type != INodeType::META) { return EISDIR; } FileBytes* fb = new FileBytes(); pfci->read_file(fb); fi->fh = reinterpret_cast(fb); - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); return 0; } @@ -1147,16 +1124,16 @@ class FuseClientContext { * @param fi file structure shared among processes opening this file. * @return error code. 0 for success. */ - int close_file(fuse_ino_t ino, struct fuse_file_info *fi) { + int close_file(fuse_ino_t ino, struct fuse_file_info* fi) { dbg_default_trace("[{}]entering {} with ino={:x}.", gettid(), __func__, ino); void* pfb = reinterpret_cast(fi->fh); - if (pfb!=nullptr) { + if(pfb != nullptr) { delete static_cast(pfb); } return 0; - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } }; -} // namespace cascade -} // namespace derecho +} // namespace cascade +} // namespace derecho From d3a1ac42c53f9bc8ed5d5f660a2d3c1c1b7261c5 Mon Sep 17 00:00:00 2001 From: Michael Ye Date: Sun, 4 Dec 2022 18:37:11 -0500 Subject: [PATCH 02/10] style: format readme and tests --- src/service/fuse/CMakeLists.txt | 3 +- src/service/fuse/README.md | 27 ++- src/service/fuse/testfuse.py | 309 ++++++++++++++++++++++---------- src/service/fuse/util.py | 76 ++++---- 4 files changed, 273 insertions(+), 142 deletions(-) diff --git a/src/service/fuse/CMakeLists.txt b/src/service/fuse/CMakeLists.txt index 0412104d..c329ced8 100644 --- a/src/service/fuse/CMakeLists.txt +++ b/src/service/fuse/CMakeLists.txt @@ -15,7 +15,7 @@ if (${HAS_FUSE}) ) target_link_libraries(fuse_client cascade readline fuse3) set_target_properties(fuse_client PROPERTIES OUTPUT_NAME cascade_fuse_client) - + add_custom_command(TARGET fuse_client POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/testfuse.py ${CMAKE_CURRENT_BINARY_DIR}/. COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/util.py ${CMAKE_CURRENT_BINARY_DIR}/. @@ -29,4 +29,3 @@ if (${HAS_FUSE}) ) endif() - diff --git a/src/service/fuse/README.md b/src/service/fuse/README.md index a5da1b86..6cea5c51 100644 --- a/src/service/fuse/README.md +++ b/src/service/fuse/README.md @@ -1,12 +1,24 @@ # Fuse API -Cascade service supports FUSE(Filesystem in Userspace) API to access data. It is a standalone client application that links with [`libfuse`](https://github.com/libfuse/libfuse). Cascade fuse api mount the file system to interact with Cascade K/V stored objects. +Cascade service supports FUSE(Filesystem in Userspace) API to access data. It is +a standalone client application that links with +[`libfuse`](https://github.com/libfuse/libfuse). Cascade fuse api mount the file +system to interact with Cascade K/V stored objects. -If you didn't see `-- Looking for include files fuse3/fuse.h, fuse3/fuse_lowlevel.h - found`, it is probably because the libfuse support is not installed. FUSE application shall be built as an executable in the built directory `src/service/fuse/cascade_fuse_client` +If you didn't see +`-- Looking for include files fuse3/fuse.h, fuse3/fuse_lowlevel.h - found`, it +is probably because the libfuse support is not installed. FUSE application shall +be built as an executable in the built directory +`src/service/fuse/cascade_fuse_client` -Running FUSE POSIX shell application is similar to run cascade cmd client. Once the cascade service is configured and started, in the client node, running `../../fuse/cascade_fuse_client [directory_to_mount]` will mount cascade file systsem to the dedicated directory. +Running FUSE POSIX shell application is similar to run cascade cmd client. Once +the cascade service is configured and started, in the client node, running +`../../fuse/cascade_fuse_client [directory_to_mount]` will mount cascade file +systsem to the dedicated directory. -Once fuse applicatino is mounted to the directory, you can access the K/V object stored in cascade using linux shell command. The structured of the mounted file system is as following +Once fuse applicatino is mounted to the directory, you can access the K/V object +stored in cascade using linux shell command. The structured of the mounted file +system is as following ```bash . @@ -34,6 +46,7 @@ Once fuse applicatino is mounted to the directory, you can access the K/V object ``` Support READ commands: + ``` cd [dir] open directory ls [dir] list directory @@ -43,7 +56,9 @@ cat .cascade read directory metadata information ``` Limitation: + - Support only single-threaded fuse client -- Current read_file keeps a buffer of file_bytes in memory, needs further optimization to read large file +- Current read_file keeps a buffer of file_bytes in memory, needs further + optimization to read large file - New features to come: WRITE commands to have fuse client interact with cascade - File write and editing commands, managing directories commands \ No newline at end of file + File write and editing commands, managing directories commands diff --git a/src/service/fuse/testfuse.py b/src/service/fuse/testfuse.py index d1f0860a..d957b45b 100644 --- a/src/service/fuse/testfuse.py +++ b/src/service/fuse/testfuse.py @@ -1,176 +1,279 @@ #!/usr/bin/env python3 -from derecho.cascade.client import ServiceClientAPI -import subprocess +import errno +import filecmp +import getopt import multiprocessing -import threading import os -import sys,getopt -import stat +import queue import shutil -import filecmp +import stat +import subprocess +import sys import tempfile +import threading import time -import errno -import sys -import queue -from tempfile import NamedTemporaryFile from contextlib import contextmanager -from util import (wait_for_mount, compare_dirs, umount, cleanup, base_cmdline, - safe_sleep, basename, test_printcap, - fuse_proto, powerset) from os.path import join as pjoin +from tempfile import NamedTemporaryFile + +from derecho.cascade.client import ServiceClientAPI +from util import ( + base_cmdline, + basename, + cleanup, + compare_dirs, + fuse_proto, + powerset, + safe_sleep, + test_printcap, + umount, + wait_for_mount, +) TEST_FILE = __file__ -with open(TEST_FILE, 'rb') as fh: +with open(TEST_FILE, "rb") as fh: TEST_DATA = fh.read() + def name_generator(__ctr=[0]): __ctr[0] += 1 - return 'testfile_%d' % __ctr[0] + return "testfile_%d" % __ctr[0] + options = [] -if sys.platform == 'linux': - options.append('clone_fd') +if sys.platform == "linux": + options.append("clone_fd") def invoke_directly(mnt_dir, name, options): - cmdline = base_cmdline + [ pjoin(basename, 'example', name), - '-f', mnt_dir, '-o', ','.join(options) ] - if name == 'cascade_fuse_client': + cmdline = base_cmdline + [ + pjoin(basename, "example", name), + "-f", + mnt_dir, + "-o", + ",".join(options), + ] + if name == "cascade_fuse_client": # supports single-threading only - cmdline.append('-s') + cmdline.append("-s") return cmdline def readdir_inode(dir): - cmd = base_cmdline + [ pjoin(basename, 'test', 'readdir_inode'), dir ] - with subprocess.Popen(cmd, stdout=subprocess.PIPE, - universal_newlines=True) as proc: + cmd = base_cmdline + [pjoin(basename, "test", "readdir_inode"), dir] + with subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) as proc: lines = proc.communicate()[0].splitlines() lines.sort() return lines -def error_msg( run_id, testcase, output, expected): +def error_msg(run_id, testcase, output, expected): msg = "[" + run_id + "]" + testcase + "(output--expected):" - msg += "(" + ','.join(output) - msg += " -- " + ','.join(expected) + ")" + msg += "(" + ",".join(output) + msg += " -- " + ",".join(expected) + ")" return msg - -put_keys = ['key0', '/a/a1/key0', '/b/b1/key1', '/c/c1/c2/key2'] -put_values = ['00', '00000000', '11111111', '22222222'] -vcss_expected_keys = [".cascade","key-key0", "key-\\a\\a1\\key0"] -pcss_expected_keys = [".cascade","key-\\b\\b1\\key1","key-\\c\\c1\\c2\\key2"] +put_keys = ["key0", "/a/a1/key0", "/b/b1/key1", "/c/c1/c2/key2"] +put_values = ["00", "00000000", "11111111", "22222222"] + +vcss_expected_keys = [".cascade", "key-key0", "key-\\a\\a1\\key0"] +pcss_expected_keys = [".cascade", "key-\\b\\b1\\key1", "key-\\c\\c1\\c2\\key2"] objp_expected_second_level = [".cascade", "a", "b", "c"] -objp_expected_third_level = [[], [".cascade", "a1"], [".cascade", "b1"], [".cascade", "c1"]] -objp_expected_keys = [[], [".cascade", "key0"], [".cascade", "key1"], [".cascade", "c2"]] +objp_expected_third_level = [ + [], + [".cascade", "a1"], + [".cascade", "b1"], + [".cascade", "c1"], +] +objp_expected_keys = [ + [], + [".cascade", "key0"], + [".cascade", "key1"], + [".cascade", "c2"], +] objp_expected_keys2 = [".cascade", "key2"] - def initial_setup(): capi = ServiceClientAPI() print("----------- CASCADE INITIAL SETUP -----------") - capi.put(put_keys[0], bytes(put_values[0],'utf-8'), subgroup_type='VolatileCascadeStoreWithStringKey', subgroup_index=0, shard_index=0) - capi.create_object_pool('/a/a1', 'VolatileCascadeStoreWithStringKey', 0) - capi.create_object_pool("/b/b1",'PersistentCascadeStoreWithStringKey', 0) - capi.create_object_pool("/c/c1/c2",'PersistentCascadeStoreWithStringKey', 0) + capi.put( + put_keys[0], + bytes(put_values[0], "utf-8"), + subgroup_type="VolatileCascadeStoreWithStringKey", + subgroup_index=0, + shard_index=0, + ) + capi.create_object_pool("/a/a1", "VolatileCascadeStoreWithStringKey", 0) + capi.create_object_pool("/b/b1", "PersistentCascadeStoreWithStringKey", 0) + capi.create_object_pool("/c/c1/c2", "PersistentCascadeStoreWithStringKey", 0) time.sleep(5) - - capi.put(put_keys[1], bytes(put_values[1],'utf-8'),blocking=True) - capi.put(put_keys[2],bytes(put_values[2],'utf-8'),blocking=True) - res = capi.put(put_keys[3], bytes(put_values[3],'utf-8'),blocking=True) + + capi.put(put_keys[1], bytes(put_values[1], "utf-8"), blocking=True) + capi.put(put_keys[2], bytes(put_values[2], "utf-8"), blocking=True) + res = capi.put(put_keys[3], bytes(put_values[3], "utf-8"), blocking=True) if res: odict = res.get_result() # print("------- THRID put result: " + f"{str(odict)}") - + # Test 1. check the directory for Cascade subgroups def tst_sg_dirs(mnt_dir, run_id=""): # First-level(subgroup_type) check mnt_first_level = os.listdir(mnt_dir) - expected_first_level = ['.cascade', 'MetadataService', 'PersistentCascadeStoreWithStringKey', 'VolatileCascadeStoreWithStringKey', 'TriggerCascadeNoStoreWithStringKey','ObjectPools' ] - error_m = error_msg(run_id, "first level", mnt_first_level, expected_first_level) + expected_first_level = [ + ".cascade", + "MetadataService", + "PersistentCascadeStoreWithStringKey", + "VolatileCascadeStoreWithStringKey", + "TriggerCascadeNoStoreWithStringKey", + "ObjectPools", + ] + error_m = error_msg(run_id, "first level", mnt_first_level, expected_first_level) assert compare_dirs(mnt_first_level, expected_first_level), error_m sg_expected_second_level = ["subgroup-0"] sg_expected_third_level = ["shard-0"] # Second-level(subgroup_index) check for subgroup_type in expected_first_level: - if(subgroup_type=='.cascade' or subgroup_type=='MetadataService' or subgroup_type=='ObjectPools'): + if ( + subgroup_type == ".cascade" + or subgroup_type == "MetadataService" + or subgroup_type == "ObjectPools" + ): continue mnt_second_level = os.listdir(mnt_dir + "/" + subgroup_type) assert compare_dirs(mnt_second_level, sg_expected_second_level) # Third-level(shard) check - mnt_third_level = os.listdir(mnt_dir + "/" + subgroup_type + "/" + sg_expected_second_level[0]) + mnt_third_level = os.listdir( + mnt_dir + "/" + subgroup_type + "/" + sg_expected_second_level[0] + ) assert compare_dirs(mnt_third_level, sg_expected_third_level) # Forth-level(key) check - if(subgroup_type == 'PersistentCascadeStoreWithStringKey'): - mnt_forth_level = os.listdir(mnt_dir + "/" + subgroup_type + "/" + sg_expected_second_level[0] + "/" + sg_expected_third_level[0]) - error_m = error_msg(run_id, "PCSS forthlevel", mnt_forth_level, pcss_expected_keys) + if subgroup_type == "PersistentCascadeStoreWithStringKey": + mnt_forth_level = os.listdir( + mnt_dir + + "/" + + subgroup_type + + "/" + + sg_expected_second_level[0] + + "/" + + sg_expected_third_level[0] + ) + error_m = error_msg( + run_id, "PCSS forthlevel", mnt_forth_level, pcss_expected_keys + ) assert compare_dirs(mnt_forth_level, pcss_expected_keys), error_m - if(subgroup_type == 'VolatileCascadeStoreWithStringKey'): - mnt_forth_level = os.listdir(mnt_dir + "/" + subgroup_type + "/" + sg_expected_second_level[0] + "/" + sg_expected_third_level[0]) - assert compare_dirs(mnt_forth_level, vcss_expected_keys) - print("["+ run_id +"]"+ "--- Test1 SubgroupTypes directories pass! ---") + if subgroup_type == "VolatileCascadeStoreWithStringKey": + mnt_forth_level = os.listdir( + mnt_dir + + "/" + + subgroup_type + + "/" + + sg_expected_second_level[0] + + "/" + + sg_expected_third_level[0] + ) + assert compare_dirs(mnt_forth_level, vcss_expected_keys) + print("[" + run_id + "]" + "--- Test1 SubgroupTypes directories pass! ---") + - - # Test 2. check the directory for Cascade object_pools +# Test 2. check the directory for Cascade object_pools def tst_objp_dirs(mnt_dir, run_id=""): mnt_objp_root = os.listdir(mnt_dir + "/" + "ObjectPools") - error_m = error_msg(run_id, "object pool test", mnt_objp_root, objp_expected_second_level) + error_m = error_msg( + run_id, "object pool test", mnt_objp_root, objp_expected_second_level + ) assert compare_dirs(mnt_objp_root, objp_expected_second_level), error_m - for i in range(1,4): - mnt_objp_dir = os.listdir(mnt_dir + "/" + "ObjectPools" + "/" + objp_expected_second_level[i]) - error_m = error_msg(run_id, f"object pool test{str(i)}", mnt_objp_dir, objp_expected_third_level[i]) + for i in range(1, 4): + mnt_objp_dir = os.listdir( + mnt_dir + "/" + "ObjectPools" + "/" + objp_expected_second_level[i] + ) + error_m = error_msg( + run_id, + f"object pool test{str(i)}", + mnt_objp_dir, + objp_expected_third_level[i], + ) assert compare_dirs(mnt_objp_dir, objp_expected_third_level[i]) - mnt_objp_key = os.listdir(mnt_dir + "/" + "ObjectPools" + "/" + objp_expected_second_level[i] + "/" + objp_expected_third_level[i][1]) + mnt_objp_key = os.listdir( + mnt_dir + + "/" + + "ObjectPools" + + "/" + + objp_expected_second_level[i] + + "/" + + objp_expected_third_level[i][1] + ) assert compare_dirs(mnt_objp_key, objp_expected_keys[i]) - if(i == 3): - mnt_objp_subdir = os.listdir(mnt_dir + "/" + "ObjectPools" + "/" + objp_expected_second_level[i] + "/" + objp_expected_third_level[i][1] + "/" + objp_expected_keys[i][1]) + if i == 3: + mnt_objp_subdir = os.listdir( + mnt_dir + + "/" + + "ObjectPools" + + "/" + + objp_expected_second_level[i] + + "/" + + objp_expected_third_level[i][1] + + "/" + + objp_expected_keys[i][1] + ) assert compare_dirs(mnt_objp_subdir, objp_expected_keys2) print("[" + run_id + "]" + "--- Test2 ObjectPools directories pass! ---") - + # Test 3. check the open and read content of files def tst_content(mnt_dir, run_id=""): # 1. subgroup content test subgroup_type = "/VolatileCascadeStoreWithStringKey/" - for i in range(1,2): - sg_mnt_key = mnt_dir + subgroup_type + "subgroup-0/shard-0/" +vcss_expected_keys[i+1] - with open(sg_mnt_key, 'r') as fh: + for i in range(1, 2): + sg_mnt_key = ( + mnt_dir + subgroup_type + "subgroup-0/shard-0/" + vcss_expected_keys[i + 1] + ) + with open(sg_mnt_key, "r") as fh: output = fh.read() - error_m = error_msg(run_id, f"sg content{sg_mnt_key}", output, put_values[i]) + error_m = error_msg( + run_id, f"sg content{sg_mnt_key}", output, put_values[i] + ) assert output == put_values[i], error_m subgroup_type = "/PersistentCascadeStoreWithStringKey/" - for i in range(2,4): - sg_mnt_key = mnt_dir + subgroup_type + "subgroup-0/shard-0/" +pcss_expected_keys[i-1] - with open(sg_mnt_key, 'r') as fh: + for i in range(2, 4): + sg_mnt_key = ( + mnt_dir + subgroup_type + "subgroup-0/shard-0/" + pcss_expected_keys[i - 1] + ) + with open(sg_mnt_key, "r") as fh: output = fh.read() - error_m = error_msg(run_id, f"sg content{sg_mnt_key}", output, put_values[i]) + error_m = error_msg( + run_id, f"sg content{sg_mnt_key}", output, put_values[i] + ) assert output == put_values[i], error_m # 2. Objectpool content test object_pool_dir = "/ObjectPools" - for i in range(1,4): + for i in range(1, 4): objp_mnt_key = mnt_dir + object_pool_dir + put_keys[i] - with open(objp_mnt_key, 'r') as fh: + with open(objp_mnt_key, "r") as fh: output = fh.read() - error_m = error_msg(run_id, f"objp content{objp_mnt_key}", output, put_values[i]) + error_m = error_msg( + run_id, f"objp content{objp_mnt_key}", output, put_values[i] + ) assert output == put_values[i], error_m - print("["+ run_id +"]" + "--- Test3 read content pass! ---") + print("[" + run_id + "]" + "--- Test3 read content pass! ---") + # def tst_attr(mnt_dir): - + + def cascade_fuse_mount(mnt_dir): # 1. create mounting piont subprocess.Popen(["mkdir", mnt_dir]) - cmd_input = base_cmdline + [ pjoin(basename, "cascade_fuse_client"), mnt_dir, "-f"] + cmd_input = base_cmdline + [pjoin(basename, "cascade_fuse_client"), mnt_dir, "-f"] # 2. run cascade_fuse_client - mount_process = subprocess.Popen(cmd_input, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + mount_process = subprocess.Popen( + cmd_input, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) # try: wait_for_mount(mount_process, mnt_dir) # except: @@ -179,18 +282,19 @@ def cascade_fuse_mount(mnt_dir): # else: # umount(mnt_dir) + def cascade_fuse_umount(mnt_dir): # 1. umount the mounting process at mnt_dir umount(mnt_dir) # 2. delete the mounted directory - subprocess.Popen(["rm", "-r",mnt_dir]) + subprocess.Popen(["rm", "-r", mnt_dir]) print(" ~~~~~~~~~~~~~ FINISHED cascade_fuse_client unmount ~~~~~~~~~~~\n") def fuse_test_cases(mnt_dir, run_id): - tst_sg_dirs(mnt_dir,run_id) - tst_objp_dirs(mnt_dir,run_id) - tst_content(mnt_dir,run_id) + tst_sg_dirs(mnt_dir, run_id) + tst_objp_dirs(mnt_dir, run_id) + tst_content(mnt_dir, run_id) # tst_attr(mnt) @@ -199,33 +303,41 @@ def main(argv): process_num = 1 thread_num = 1 try: - opts, args = getopt.getopt(argv,"ht:p:",["threadN=","processN="]) + opts, args = getopt.getopt(argv, "ht:p:", ["threadN=", "processN="]) except getopt.GetoptError: - print('fusetest.py -t -p ') + print("fusetest.py -t -p ") sys.exit(2) for opt, arg in opts: - if opt == '-h': - print ('fusetest.py -t -p ') + if opt == "-h": + print("fusetest.py -t -p ") sys.exit() elif opt in ("-t", "--ifile"): thread_num = int(arg) elif opt in ("-p", "--ofile"): process_num = int(arg) - + cascade_process = multiprocessing.Process(target=initial_setup) cascade_process.start() cascade_process.join() - mnt_dir = pjoin(basename,"testdir") # use build/src/service/fuse/testdir as mounting point + mnt_dir = pjoin( + basename, "testdir" + ) # use build/src/service/fuse/testdir as mounting point fuse_process = multiprocessing.Process(target=cascade_fuse_mount, args=(mnt_dir,)) fuse_process.start() - fuse_process.join() # used when fuse not run in foreground in fuse_client.cpp - + fuse_process.join() # used when fuse not run in foreground in fuse_client.cpp + # multi-processing test cases test_processes = [] for i in range(process_num): run_id = "process_" + str(i) - proc = multiprocessing.Process(target=fuse_test_cases, args=(mnt_dir,run_id,)) + proc = multiprocessing.Process( + target=fuse_test_cases, + args=( + mnt_dir, + run_id, + ), + ) test_processes.append(proc) proc.start() for proc in test_processes: @@ -235,7 +347,13 @@ def main(argv): test_threads = [] for i in range(thread_num): run_id = "thread_" + str(i) - thread = threading.Thread(target=fuse_test_cases, args=(mnt_dir,run_id,)) + thread = threading.Thread( + target=fuse_test_cases, + args=( + mnt_dir, + run_id, + ), + ) test_threads.append(thread) thread.start() for thread in test_threads: @@ -244,5 +362,6 @@ def main(argv): # fuse_process.terminate() cascade_fuse_umount(mnt_dir) + if __name__ == "__main__": - main(sys.argv[1:]) + main(sys.argv[1:]) diff --git a/src/service/fuse/util.py b/src/service/fuse/util.py index 45b239a8..1548b870 100644 --- a/src/service/fuse/util.py +++ b/src/service/fuse/util.py @@ -1,48 +1,47 @@ #!/usr/bin/env python3 -import subprocess -import pytest +import itertools import os +import re import stat +import subprocess +import sys import time from os.path import join as pjoin -import sys -import re -import itertools + +import pytest basename = os.path.dirname(os.path.abspath(__file__)) fusermount3_dir = "/home/yy354/opt-dev/bin" def test_printcap(): - cmdline = base_cmdline + [ pjoin(basename, 'example', 'printcap') ] - proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE, - universal_newlines=True) + cmdline = base_cmdline + [pjoin(basename, "example", "printcap")] + proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE, universal_newlines=True) (stdout, _) = proc.communicate(30) assert proc.returncode == 0 proto = None caps = set() - for line in stdout.split('\n'): - if line.startswith('\t'): + for line in stdout.split("\n"): + if line.startswith("\t"): caps.add(line.strip()) continue - hit = re.match(r'Protocol version: (\d+)\.(\d+)$', line) + hit = re.match(r"Protocol version: (\d+)\.(\d+)$", line) if hit: proto = (int(hit.group(1)), int(hit.group(2))) return (proto, caps) -def wait_for_mount(mount_process, mnt_dir, - test_fn=os.path.ismount): +def wait_for_mount(mount_process, mnt_dir, test_fn=os.path.ismount): elapsed = 0 while elapsed < 20: if test_fn(mnt_dir): print(" -------- Fuse mount succeed -------") return True if mount_process.poll() is not None: - pytest.fail('file system process terminated prematurely') + pytest.fail("file system process terminated prematurely") time.sleep(0.1) elapsed += 0.1 pytest.fail("mountpoint failed to come up") @@ -56,28 +55,27 @@ def compare_dirs(output, expected): if not dir in output: return False return True - + def cleanup(mount_process, mnt_dir): # Don't bother trying Valgrind if things already went wrong - if 'bsd' in sys.platform or 'dragonfly' in sys.platform: - cmd = [ 'umount', '-f', mnt_dir ] + if "bsd" in sys.platform or "dragonfly" in sys.platform: + cmd = ["umount", "-f", mnt_dir] else: - cmd = [pjoin(basename, 'util', 'fusermount3'), - '-z', '-u', mnt_dir] - subprocess.call(cmd, stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT) + cmd = [pjoin(basename, "util", "fusermount3"), "-z", "-u", mnt_dir] + subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) mount_process.terminate() try: mount_process.wait(1) except subprocess.TimeoutExpired: mount_process.kill() + def umount(mnt_dir): - if 'bsd' in sys.platform or 'dragonfly' in sys.platform: - cmdline = [ 'umount', mnt_dir ] + if "bsd" in sys.platform or "dragonfly" in sys.platform: + cmdline = ["umount", mnt_dir] else: # fusermount3 will be setuid root, so we can only trace it with # valgrind if we're root @@ -85,21 +83,17 @@ def umount(mnt_dir): cmdline = base_cmdline else: cmdline = [] - cmdline = cmdline + [ pjoin(fusermount3_dir, 'fusermount3'), - '-z', '-u', mnt_dir ] + cmdline = cmdline + [pjoin(fusermount3_dir, "fusermount3"), "-z", "-u", mnt_dir] subprocess.check_call(cmdline) assert not os.path.ismount(mnt_dir) - - - def safe_sleep(secs): - '''Like time.sleep(), but sleep for at least *secs* + """Like time.sleep(), but sleep for at least *secs* `time.sleep` may sleep less than the given period if a signal is received. This function ensures that we sleep for at least the desired time. - ''' + """ now = time.time() end = now + secs @@ -109,26 +103,30 @@ def safe_sleep(secs): def powerset(iterable): - s = list(iterable) - return itertools.chain.from_iterable( - itertools.combinations(s, r) for r in range(len(s)+1)) + s = list(iterable) + return itertools.chain.from_iterable( + itertools.combinations(s, r) for r in range(len(s) + 1) + ) # Use valgrind if requested -if os.environ.get('TEST_WITH_VALGRIND', 'no').lower().strip() \ - not in ('no', 'false', '0'): - base_cmdline = [ 'valgrind', '-q', '--' ] +if os.environ.get("TEST_WITH_VALGRIND", "no").lower().strip() not in ( + "no", + "false", + "0", +): + base_cmdline = ["valgrind", "-q", "--"] else: base_cmdline = [] # Try to use local fusermount3 -os.environ['PATH'] = '%s:%s' % (pjoin(basename, 'util'), os.environ['PATH']) +os.environ["PATH"] = "%s:%s" % (pjoin(basename, "util"), os.environ["PATH"]) # Put example binaries on PATH -os.environ['PATH'] = '%s:%s' % (pjoin(basename, 'example'), os.environ['PATH']) +os.environ["PATH"] = "%s:%s" % (pjoin(basename, "example"), os.environ["PATH"]) try: (fuse_proto, fuse_caps) = test_printcap() except: # Rely on test to raise error - fuse_proto = (0,0) + fuse_proto = (0, 0) fuse_caps = set() From 7f58bfced3d29fea07e1cb03038f7192b1194367 Mon Sep 17 00:00:00 2001 From: Michael Ye Date: Mon, 5 Dec 2022 01:28:36 -0500 Subject: [PATCH 03/10] feat: add signal handler for server --- scripts/util/cascade_runner | 196 +++++++++++++++++++++++++++++++ src/service/fuse/README.md | 24 ++-- src/service/fuse/fuse_client.cpp | 2 +- src/service/server.cpp | 56 +++++++-- 4 files changed, 252 insertions(+), 26 deletions(-) create mode 100755 scripts/util/cascade_runner diff --git a/scripts/util/cascade_runner b/scripts/util/cascade_runner new file mode 100755 index 00000000..1d148cc2 --- /dev/null +++ b/scripts/util/cascade_runner @@ -0,0 +1,196 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") +SCRIPT_NAME=$(basename "${BASH_SOURCE[0]}") +HELP_STRING="A helper script to set up a local cascade instance +Usage: $SCRIPT_NAME [options] + -h Print this message and exit. + -d D CFG directory location (defaults to cfg). + -n N Sets a range from 0..N-1 for range commands commands. + + -u K update key in range. + -v V update value in range. + (ex. -n 7 -u default_log_level -v info) + + -k Kill running cascade nodes. + -r Remove old logs in range. + -s Start server nodes in range. + -f D directory to mount (requires -c flag to start and -k to end). + -c I Start client node at folder cfg/nI." + +CFG_DIR="$SCRIPT_DIR/cfg" +FUSE_DIR="" +HELP=false +KILL=false +CLEAN=false +SERVERS=false +NODES=0 +CLIENT=-1 + +UPDATE_KEY="" +UPDATE_VALUE="" + +# TODO get from env? or as arg +BASE_PATH="/root/workspace/cascade/build-Release/src" + +export PATH="$PATH:$BASE_PATH/service" +export PATH="$PATH:$BASE_PATH/service/fuse" + +FUSE_COMMAND="cascade_fuse_client" +CLIENT_COMMAND="cascade_client" +SERVER_COMMAND="cascade_server" +LOG_FILE="node_info.log" + +if ! command -v $SERVER_COMMAND &>/dev/null; then + echo "$SERVER_COMMAND could not be found" + exit_abnormal +fi + +usage() { + echo "$HELP_STRING" 1>&2 +} + +exit_abnormal() { + usage + exit 1 +} + +check_num() { + # $1: string to check + RE_ISNUM='^[0-9]+$' + if ! [[ $1 =~ $RE_ISNUM ]]; then + exit_abnormal + fi +} + +foreach() { + # $1: command to do (accepts count as arg) + COUNT=0 + while [ $COUNT -lt "$NODES" ]; do + DIR="$CFG_DIR/n$COUNT" + cd "$DIR" + + $1 $COUNT + + cd - &>/dev/null + ((COUNT += 1)) + done +} + +main() { + while getopts 'hd:n:f:krsc:u:v:' opt; do + case "$opt" in + h) HELP=true ;; + d) CFG_DIR="$OPTARG" ;; + n) + NODES="$OPTARG" + check_num "$NODES" + ;; + f) FUSE_DIR="$OPTARG" ;; + k) KILL=true ;; + r) CLEAN=true ;; + s) SERVERS=true ;; + c) + CLIENT="$OPTARG" + check_num "$CLIENT" + ;; + u) UPDATE_KEY="$OPTARG" ;; + v) UPDATE_VALUE="$OPTARG" ;; + *) exit_abnormal ;; + esac + done + + if [[ "$HELP" = true ]]; then + usage + exit + fi + + if [[ -n "$FUSE_DIR" ]]; then + MOUNT="$(realpath "$FUSE_DIR")" + + if [[ ! "$CLIENT" -eq "-1" ]]; then + DIR="$CFG_DIR/n$CLIENT" + cd "$DIR" + + echo "fuse client mounting at $MOUNT" + mkdir -p "$MOUNT" + "$FUSE_COMMAND" -s -f "$MOUNT" &>"$LOG_FILE" & + + cd - &>/dev/null + exit + fi + + if [[ "$KILL" = true ]]; then + echo "unmounting $MOUNT and ending fuses client" + umount "$MOUNT" + + pkill -SIGINT "$FUSE_COMMAND" || true + + exit + fi + fi + + if [[ "$KILL" = true ]]; then + # TODO better idea: save PIDs? + + pkill -SIGINT "$CLIENT_COMMAND" || true + pkill -SIGINT "$SERVER_COMMAND" || true # kill cascade_server + + # kill old cascade_runner and subprocesses (excluding itself) + # pgrep "${SCRIPT_NAME%.*}" | grep -v "$$" | while read -r LINE; do + # SUB_SLEEP=$(pgrep -P "$LINE") + # kill "$LINE" # kill cascade_runner while loop + # kill "$SUB_SLEEP" # kill sleep + # done + fi + + if [[ "$CLEAN" = true ]]; then + x() { + echo "$1 > removing .plog and logs" + rm -rf .plog derecho_debug.log "$LOG_FILE" + } + foreach x + fi + + if [[ -n "$UPDATE_KEY" ]]; then + x() { + CONFIG_FILE="derecho.cfg" + echo "$1 > updating $UPDATE_KEY with $UPDATE_VALUE in $CONFIG_FILE" + sed -i "s/\($UPDATE_KEY *= *\)\(.*\)/\1$UPDATE_VALUE/" "$CONFIG_FILE" + } + foreach x + fi + + if [[ ! "$CLIENT" -eq "-1" ]]; then + DIR="$CFG_DIR/n$CLIENT" + + if [[ -n "$FUSE_DIR" ]]; then + mkdir -p "$MOUNT" + + cd "$DIR" + "$FUSE_COMMAND" -s -f "$MOUNT" &>"$LOG_FILE" & + else + cd "$DIR" + "$CLIENT_COMMAND" + fi + + cd - &>/dev/null + exit + fi + set +e + + if [[ "$SERVERS" = true ]]; then + x() { + echo "$1 > starting server node" + "$SERVER_COMMAND" --signal &>"$LOG_FILE" & + # (while true; do sleep 10000; done) | + # "$SERVER_COMMAND" &>"$LOG_FILE" & + } + foreach x + + fi +} + +main "$@" diff --git a/src/service/fuse/README.md b/src/service/fuse/README.md index 6cea5c51..8fcb79ba 100644 --- a/src/service/fuse/README.md +++ b/src/service/fuse/README.md @@ -16,7 +16,7 @@ the cascade service is configured and started, in the client node, running `../../fuse/cascade_fuse_client [directory_to_mount]` will mount cascade file systsem to the dedicated directory. -Once fuse applicatino is mounted to the directory, you can access the K/V object +Once fuse application is mounted to the directory, you can access the K/V object stored in cascade using linux shell command. The structured of the mounted file system is as following @@ -26,21 +26,21 @@ system is as following |-- PersistentCascadeStoreWithStringKey | |-- subgroup-0 | |-- shard-0 -| |-- .cascade +| |-- .cascade | |-- key-0 | ... |-- VolatileCascadeStoreWithStringKey | |-- subgroup-0 | |-- shard-0 -| |-- .cascade +| |-- .cascade | ... |-- TriggerCascadeStoreWithStringKey |-- ObjectPools | |-- objectpoolpathname-a -| |-- a1 -| |-- objectpool object 0 -| ... -| |-- .cascade +| |-- a1 +| |-- objectpool object 0 +| ... +| |-- .cascade | |-- .cascade |-- .cascade ``` @@ -48,11 +48,11 @@ system is as following Support READ commands: ``` -cd [dir] open directory -ls [dir] list directory -ls -a list directory with attributes -cat [file] read file -cat .cascade read directory metadata information +cd [dir] open directory +ls [dir] list directory +ls -a list directory with attributes +cat [file] read file +cat .cascade read directory metadata information ``` Limitation: diff --git a/src/service/fuse/fuse_client.cpp b/src/service/fuse/fuse_client.cpp index ef270ce0..b8bfbd4a 100644 --- a/src/service/fuse/fuse_client.cpp +++ b/src/service/fuse/fuse_client.cpp @@ -278,7 +278,7 @@ int main(int argc, char** argv) { // log_current_dir(opts.foreground); - /* Block until ctrl+c or fuserount -u */ + /* Block until ctrl+c or fusermount -u */ if(opts.singlethread) { ret = fuse_session_loop(se); } else { diff --git a/src/service/server.cpp b/src/service/server.cpp index c7850d45..fb1c4960 100644 --- a/src/service/server.cpp +++ b/src/service/server.cpp @@ -1,10 +1,11 @@ #include "server.hpp" #include "cascade/cascade.hpp" +#include "cascade/object.hpp" #include "cascade/service.hpp" #include "cascade/service_types.hpp" -#include "cascade/object.hpp" +#include #include #include #include @@ -15,7 +16,36 @@ using namespace derecho::cascade; +void terminate() { + // wait for service to quit. + Service::shutdown(false); + dbg_default_trace("shutdown service gracefully"); + // you can do something here to parallel the destructing process. + Service::wait(); + dbg_default_trace("Finish shutdown."); +} + +void signal_handler(int signum) { + dbg_default_trace("received interrupt signal {}", signum); + + terminate(); + exit(signum); +} + int main(int argc, char** argv) { + // check for signal_arg + bool use_signal = false; + for(int i = 0; i < argc; ++i) { + printf("Argument %d : %s\n", i, argv[i]); + if(strcmp(argv[i], "--signal") == 0) { + use_signal = true; + } + } + // set proc name if(prctl(PR_SET_NAME, PROC_NAME, 0, 0, 0) != 0) { dbg_default_warn("Cannot set proc name to {}.", PROC_NAME); @@ -47,18 +77,18 @@ int main(int argc, char** argv) { meta_factory, vcss_factory, pcss_factory, tcss_factory); dbg_default_trace("started service, waiting till it ends."); - std::cout << "Press Enter to Shutdown." << std::endl; - std::cin.get(); - // wait for service to quit. - Service::shutdown(false); - dbg_default_trace("shutdown service gracefully"); - // you can do something here to parallel the destructing process. - Service::wait(); - dbg_default_trace("Finish shutdown."); + + if(use_signal) { + printf("Send SIGINT (Ctrl+C) to Shutdown.\n"); + signal(SIGINT, signal_handler); + while(true) { + sleep(60); + } + } else { + printf("Press Enter to Shutdown.\n"); + std::cin.get(); + terminate(); + } return 0; } From 8dcd413e0c3eb696ae471149aed6c98b5a1fde07 Mon Sep 17 00:00:00 2001 From: Michael Ye Date: Wed, 11 Jan 2023 21:11:57 -0500 Subject: [PATCH 04/10] feat: add fuse starter write logic and change signal handling --- include/cascade/object.hpp | 217 ++++++++++++----------- scripts/util/cascade_runner | 24 ++- src/service/fuse/CMakeLists.txt | 9 +- src/service/fuse/fuse_client.cpp | 140 ++++++++------- src/service/fuse/fuse_client_context.hpp | 196 +++++++++++++++++--- src/service/fuse/fuse_client_signals.cpp | 49 +++++ src/service/fuse/fuse_client_signals.hpp | 7 + 7 files changed, 442 insertions(+), 200 deletions(-) create mode 100644 src/service/fuse/fuse_client_signals.cpp create mode 100644 src/service/fuse/fuse_client_signals.hpp diff --git a/include/cascade/object.hpp b/include/cascade/object.hpp index 997c0357..18fd9a38 100644 --- a/include/cascade/object.hpp +++ b/include/cascade/object.hpp @@ -1,14 +1,15 @@ #pragma once #include +#include #include #include #include +#include #include #include #include -#include -#include #include +#include #include #include @@ -20,8 +21,8 @@ using namespace persistent; using namespace std::chrono_literals; -namespace derecho{ -namespace cascade{ +namespace derecho { +namespace cascade { enum object_memory_mode_t { DEFAULT, @@ -29,7 +30,7 @@ enum object_memory_mode_t { BLOB_GENERATOR, }; -using blob_generator_func_t = std::function; +using blob_generator_func_t = std::function; class Blob : public mutils::ByteRepresentable { public: @@ -40,8 +41,7 @@ class Blob : public mutils::ByteRepresentable { // for BLOB_GENERATOR mode only blob_generator_func_t blob_generator; - object_memory_mode_t memory_mode; - + object_memory_mode_t memory_mode; // constructor - copy to own the data Blob(const uint8_t* const b, const decltype(size) s); @@ -81,53 +81,54 @@ class Blob : public mutils::ByteRepresentable { static std::unique_ptr from_bytes(mutils::DeserializationManager*, const uint8_t* const v); static mutils::context_ptr from_bytes_noalloc( - mutils::DeserializationManager* ctx, - const uint8_t* const v); + mutils::DeserializationManager* ctx, + const uint8_t* const v); static mutils::context_ptr from_bytes_noalloc_const( - mutils::DeserializationManager* ctx, - const uint8_t* const v); + mutils::DeserializationManager* ctx, + const uint8_t* const v); }; #define INVALID_UINT64_OBJECT_KEY (0xffffffffffffffffLLU) class ObjectWithUInt64Key : public mutils::ByteRepresentable, - public ICascadeObject, + public ICascadeObject, public IKeepTimestamp, public IVerifyPreviousVersion #ifdef ENABLE_EVALUATION - , public IHasMessageID + , + public IHasMessageID #endif - { +{ public: #ifdef ENABLE_EVALUATION - mutable uint64_t message_id; + mutable uint64_t message_id; #endif - mutable persistent::version_t version; - mutable uint64_t timestamp_us; - mutable persistent::version_t previous_version; // previous version, INVALID_VERSION for the first version - mutable persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. - uint64_t key; // object_id - Blob blob; // the object + mutable persistent::version_t version; + mutable uint64_t timestamp_us; + mutable persistent::version_t previous_version; // previous version, INVALID_VERSION for the first version + mutable persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. + uint64_t key; // object_id + Blob blob; // the object // bool operator==(const ObjectWithUInt64Key& other); // constructor 0 : copy constructor - ObjectWithUInt64Key(const uint64_t _key, + ObjectWithUInt64Key(const uint64_t _key, const Blob& _blob); // constructor 0.5 : copy/emplace constructor ObjectWithUInt64Key( #ifdef ENABLE_EVALUATION - const uint64_t _message_id, + const uint64_t _message_id, #endif - const persistent::version_t _version, - const uint64_t _timestamp_us, - const persistent::version_t _previous_version, - const persistent::version_t _previous_version_by_key, - const uint64_t _key, - const Blob& _blob, - bool is_emplaced = false); + const persistent::version_t _version, + const uint64_t _timestamp_us, + const persistent::version_t _previous_version, + const persistent::version_t _previous_version_by_key, + const uint64_t _key, + const Blob& _blob, + bool is_emplaced = false); // constructor 1 : copy constructor ObjectWithUInt64Key(const uint64_t _key, @@ -137,15 +138,15 @@ class ObjectWithUInt64Key : public mutils::ByteRepresentable, // constructor 1.5 : copy constructor ObjectWithUInt64Key( #ifdef ENABLE_EVALUATION - const uint64_t _message_id, + const uint64_t _message_id, #endif - const persistent::version_t _version, - const uint64_t _timestamp_us, - const persistent::version_t _previous_version, - const persistent::version_t _previous_version_by_key, - const uint64_t _key, - const uint8_t* const _b, - const std::size_t _s); + const persistent::version_t _version, + const uint64_t _timestamp_us, + const persistent::version_t _previous_version, + const persistent::version_t _previous_version_by_key, + const uint64_t _key, + const uint8_t* const _b, + const std::size_t _s); // TODO: we need a move version for the deserializer. @@ -165,15 +166,15 @@ class ObjectWithUInt64Key : public mutils::ByteRepresentable, // constructor 5.5 : using delayed instratiator with message generator ObjectWithUInt64Key( #ifdef ENABLE_EVALUATION - const uint64_t _message_id, + const uint64_t _message_id, #endif - const persistent::version_t _version, - const uint64_t _timestamp_us, - const persistent::version_t _previous_version, - const persistent::version_t _previous_version_by_key, - const uint64_t _key, - const blob_generator_func_t& _message_generator, - const std::size_t _s); + const persistent::version_t _version, + const uint64_t _timestamp_us, + const persistent::version_t _previous_version, + const persistent::version_t _previous_version_by_key, + const uint64_t _key, + const blob_generator_func_t& _message_generator, + const std::size_t _s); virtual const uint64_t& get_key_ref() const override; virtual bool is_null() const override; @@ -198,11 +199,11 @@ class ObjectWithUInt64Key : public mutils::ByteRepresentable, void ensure_registerd(mutils::DeserializationManager&) {} static std::unique_ptr from_bytes(mutils::DeserializationManager*, const uint8_t* const v); static mutils::context_ptr from_bytes_noalloc( - mutils::DeserializationManager* ctx, - const uint8_t* const v); + mutils::DeserializationManager* ctx, + const uint8_t* const v); static mutils::context_ptr from_bytes_noalloc_const( - mutils::DeserializationManager* ctx, - const uint8_t* const v); + mutils::DeserializationManager* ctx, + const uint8_t* const v); // IK and IV for volatile cascade store static uint64_t IK; @@ -211,66 +212,68 @@ class ObjectWithUInt64Key : public mutils::ByteRepresentable, inline std::ostream& operator<<(std::ostream& out, const Blob& b) { out << "[size:" << b.size << ", data:" << std::hex; - if(b.size > 0) { - uint32_t i = 0; - for(i = 0; i < 8 && i < b.size; i++) { - out << " " << b.bytes[i]; - } - if(i < b.size) { - out << "..."; - } - } + // if(b.size > 0) { + // uint32_t i = 0; + // for(i = 0; i < 8 && i < b.size; i++) { + // out << " " << b.bytes[i]; + // } + // if(i < b.size) { + // out << "..."; + // } + // } + out << std::quoted(reinterpret_cast(b.bytes)); out << std::dec << "]"; return out; } inline std::ostream& operator<<(std::ostream& out, const ObjectWithUInt64Key& o) { - out << "ObjectWithUInt64Key{ver: 0x" << std::hex << o.version << std::dec - << ", ts(us): " << o.timestamp_us + out << "ObjectWithUInt64Key{ver: 0x" << std::hex << o.version << std::dec + << ", ts(us): " << o.timestamp_us << ", prev_ver: " << std::hex << o.previous_version << std::dec << ", prev_ver_by_key: " << std::hex << o.previous_version_by_key << std::dec - << ", id:" << o.key + << ", id:" << o.key << ", data:" << o.blob << "}"; return out; } class ObjectWithStringKey : public mutils::ByteRepresentable, - public ICascadeObject, + public ICascadeObject, public IKeepTimestamp, public IVerifyPreviousVersion #ifdef ENABLE_EVALUATION - ,public IHasMessageID + , + public IHasMessageID #endif - { +{ public: #ifdef ENABLE_EVALUATION - mutable uint64_t message_id; + mutable uint64_t message_id; #endif - mutable persistent::version_t version; // object version - mutable uint64_t timestamp_us; // timestamp in microsecond - mutable persistent::version_t previous_version; // previous version, INVALID_VERSION for the first version. - mutable persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. - std::string key; // object_id - Blob blob; // the object data + mutable persistent::version_t version; // object version + mutable uint64_t timestamp_us; // timestamp in microsecond + mutable persistent::version_t previous_version; // previous version, INVALID_VERSION for the first version. + mutable persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. + std::string key; // object_id + Blob blob; // the object data // bool operator==(const ObjectWithStringKey& other); // constructor 0 : copy constructor - ObjectWithStringKey(const std::string& _key, + ObjectWithStringKey(const std::string& _key, const Blob& _blob); // constructor 0.5 : copy/in-place constructor ObjectWithStringKey( #ifdef ENABLE_EVALUATION - const uint64_t message_id, + const uint64_t message_id, #endif - const persistent::version_t _version, - const uint64_t _timestamp_us, - const persistent::version_t _previous_version, - const persistent::version_t _previous_version_by_key, - const std::string& _key, - const Blob& _blob, - bool is_emplaced = false); + const persistent::version_t _version, + const uint64_t _timestamp_us, + const persistent::version_t _previous_version, + const persistent::version_t _previous_version_by_key, + const std::string& _key, + const Blob& _blob, + bool is_emplaced = false); // constructor 1 : copy consotructor ObjectWithStringKey(const std::string& _key, @@ -280,15 +283,15 @@ class ObjectWithStringKey : public mutils::ByteRepresentable, // constructor 1.5 : copy constructor ObjectWithStringKey( #ifdef ENABLE_EVALUATION - const uint64_t message_id, + const uint64_t message_id, #endif - const persistent::version_t _version, - const uint64_t _timestamp_us, - const persistent::version_t _previous_version, - const persistent::version_t _previous_version_by_key, - const std::string& _key, - const uint8_t* const _b, - const std::size_t _s); + const persistent::version_t _version, + const uint64_t _timestamp_us, + const persistent::version_t _previous_version, + const persistent::version_t _previous_version_by_key, + const std::string& _key, + const uint8_t* const _b, + const std::size_t _s); // TODO: we need a move version for the deserializer. @@ -308,15 +311,15 @@ class ObjectWithStringKey : public mutils::ByteRepresentable, // constructor 5.5 : using delayed instatiator withe message generator ObjectWithStringKey( #ifdef ENABLE_EVALUATION - const uint64_t message_id, + const uint64_t message_id, #endif - const persistent::version_t _version, - const uint64_t _timestamp_us, - const persistent::version_t _previous_version, - const persistent::version_t _previous_version_by_key, - const std::string& _key, - const blob_generator_func_t& _message_generator, - const std::size_t _s); + const persistent::version_t _version, + const uint64_t _timestamp_us, + const persistent::version_t _previous_version, + const persistent::version_t _previous_version_by_key, + const std::string& _key, + const blob_generator_func_t& _message_generator, + const std::size_t _s); virtual const std::string& get_key_ref() const override; virtual bool is_null() const override; @@ -333,18 +336,18 @@ class ObjectWithStringKey : public mutils::ByteRepresentable, virtual uint64_t get_message_id() const override; #endif -// DEFAULT_SERIALIZATION_SUPPORT(ObjectWithStringKey, version, timestamp_us, previous_version, previous_version_by_key, key, blob); + // DEFAULT_SERIALIZATION_SUPPORT(ObjectWithStringKey, version, timestamp_us, previous_version, previous_version_by_key, key, blob); std::size_t to_bytes(uint8_t* v) const; std::size_t bytes_size() const; void post_object(const std::function& f) const; void ensure_registerd(mutils::DeserializationManager&) {} static std::unique_ptr from_bytes(mutils::DeserializationManager*, const uint8_t* const v); static mutils::context_ptr from_bytes_noalloc( - mutils::DeserializationManager* ctx, - const uint8_t* const v); + mutils::DeserializationManager* ctx, + const uint8_t* const v); static mutils::context_ptr from_bytes_noalloc_const( - mutils::DeserializationManager* ctx, - const uint8_t* const v); + mutils::DeserializationManager* ctx, + const uint8_t* const v); // IK and IV for volatile cascade store static std::string IK; @@ -352,15 +355,15 @@ class ObjectWithStringKey : public mutils::ByteRepresentable, }; inline std::ostream& operator<<(std::ostream& out, const ObjectWithStringKey& o) { - out << "ObjectWithStringKey{" + out << "ObjectWithStringKey{" #ifdef ENABLE_EVALUATION << "msg_id: " << o.message_id #endif - << "ver: 0x" << std::hex << o.version << std::dec + << "ver: 0x" << std::hex << o.version << std::dec << ", ts: " << o.timestamp_us << ", prev_ver: " << std::hex << o.previous_version << std::dec << ", prev_ver_by_key: " << std::hex << o.previous_version_by_key << std::dec - << ", id:" << o.key + << ", id:" << o.key << ", data:" << o.blob << "}"; return out; } @@ -372,5 +375,5 @@ std::enable_if_t,std::is_s } **/ -} // namespace cascade -} // namespace derecho +} // namespace cascade +} // namespace derecho diff --git a/scripts/util/cascade_runner b/scripts/util/cascade_runner index 1d148cc2..8aca3924 100755 --- a/scripts/util/cascade_runner +++ b/scripts/util/cascade_runner @@ -8,7 +8,7 @@ HELP_STRING="A helper script to set up a local cascade instance Usage: $SCRIPT_NAME [options] -h Print this message and exit. -d D CFG directory location (defaults to cfg). - -n N Sets a range from 0..N-1 for range commands commands. + -n N Sets a range from 0..N-1 for range commands. -u K update key in range. -v V update value in range. @@ -18,9 +18,11 @@ Usage: $SCRIPT_NAME [options] -r Remove old logs in range. -s Start server nodes in range. -f D directory to mount (requires -c flag to start and -k to end). + -F D directory to mount (runs in foreground). -c I Start client node at folder cfg/nI." CFG_DIR="$SCRIPT_DIR/cfg" +FUSE_FOREGROUND=false FUSE_DIR="" HELP=false KILL=false @@ -80,7 +82,7 @@ foreach() { } main() { - while getopts 'hd:n:f:krsc:u:v:' opt; do + while getopts 'hd:n:f:F:krsc:u:v:' opt; do case "$opt" in h) HELP=true ;; d) CFG_DIR="$OPTARG" ;; @@ -89,6 +91,10 @@ main() { check_num "$NODES" ;; f) FUSE_DIR="$OPTARG" ;; + F) + FUSE_DIR="$OPTARG" + FUSE_FOREGROUND=true + ;; k) KILL=true ;; r) CLEAN=true ;; s) SERVERS=true ;; @@ -116,7 +122,12 @@ main() { echo "fuse client mounting at $MOUNT" mkdir -p "$MOUNT" - "$FUSE_COMMAND" -s -f "$MOUNT" &>"$LOG_FILE" & + + if [[ "$FUSE_FOREGROUND" = true ]]; then + "$FUSE_COMMAND" -s -f "$MOUNT" + else + "$FUSE_COMMAND" -s -f "$MOUNT" &>"$LOG_FILE" & + fi cd - &>/dev/null exit @@ -166,12 +177,7 @@ main() { if [[ ! "$CLIENT" -eq "-1" ]]; then DIR="$CFG_DIR/n$CLIENT" - if [[ -n "$FUSE_DIR" ]]; then - mkdir -p "$MOUNT" - - cd "$DIR" - "$FUSE_COMMAND" -s -f "$MOUNT" &>"$LOG_FILE" & - else + if [[ -z "$FUSE_DIR" ]]; then cd "$DIR" "$CLIENT_COMMAND" fi diff --git a/src/service/fuse/CMakeLists.txt b/src/service/fuse/CMakeLists.txt index c329ced8..6770ff1d 100644 --- a/src/service/fuse/CMakeLists.txt +++ b/src/service/fuse/CMakeLists.txt @@ -6,6 +6,13 @@ set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) find_package(derecho CONFIG REQUIRED) if (${HAS_FUSE}) + add_library(fuse_client_signals OBJECT fuse_client_signals.cpp) + target_include_directories(fuse_client_signals PRIVATE + $ + $ + $ + ) + add_executable(fuse_client fuse_client.cpp) target_include_directories(fuse_client PRIVATE $ @@ -13,7 +20,7 @@ if (${HAS_FUSE}) $ $ ) - target_link_libraries(fuse_client cascade readline fuse3) + target_link_libraries(fuse_client cascade readline fuse3 fuse_client_signals) set_target_properties(fuse_client PROPERTIES OUTPUT_NAME cascade_fuse_client) add_custom_command(TARGET fuse_client POST_BUILD diff --git a/src/service/fuse/fuse_client.cpp b/src/service/fuse/fuse_client.cpp index b8bfbd4a..bf9e0c73 100644 --- a/src/service/fuse/fuse_client.cpp +++ b/src/service/fuse/fuse_client.cpp @@ -1,20 +1,21 @@ -#define FUSE_USE_VERSION 31 -#include "fuse_client_context.hpp" #include #include #include -#include + +#include #include #include #include #include #include -#include +#include "fuse_client_context.hpp" +#include "fuse_client_signals.hpp" + #define FUSE_CLIENT_DEV_ID (0xCA7CADE) /** - * fuse_client mount the cascade service to file system. This allows users to access cascade data with normal POSIX + * fuse_client mounts the cascade service as a file system. This allows users to access cascade data with normal POSIX * filesystem API. * * The data in cascade is organized this way: @@ -102,11 +103,17 @@ static void dirbuf_add(fuse_req_t req, struct dirbuf* b, const char* name, fuse_ b->p = (char*)realloc(b->p, b->size); std::memset(&stbuf, 0, sizeof(stbuf)); stbuf.st_ino = ino; - FCC_REQ(req)->fill_stbuf_by_ino(stbuf); + FCC_REQ(req)->fill_stbuf_by_ino(stbuf); // additional effect fuse_add_direntry(req, b->p + oldsize, b->size - oldsize, name, &stbuf, b->size); } -#define min(x, y) ((x) < (y) ? (x) : (y)) +static int reply_buf_limited(fuse_req_t req, const char* buf, size_t bufsize, + off_t off, size_t maxsize) { + if(static_cast(off) < bufsize) + return fuse_reply_buf(req, buf + off, std::min(bufsize - off, maxsize)); + else + return fuse_reply_buf(req, NULL, 0); +} static void fs_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, struct fuse_file_info* fi) { dbg_default_trace("entering {}.", __func__); @@ -117,11 +124,7 @@ static void fs_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, s for(auto kv : FCC_REQ(req)->get_dir_entries(ino)) { dirbuf_add(req, &b, kv.first.c_str(), kv.second); } - if(static_cast(off) < b.size) { - fuse_reply_buf(req, b.p + off, min(b.size - off, size)); - } else { - fuse_reply_buf(req, NULL, 0); - } + reply_buf_limited(req, b.p, b.size, off, size); free(b.p); dbg_default_trace("leaving {}.", __func__); } @@ -129,7 +132,8 @@ static void fs_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, s static void fs_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { dbg_default_trace("entering {}.", __func__); int err; - if((fi->flags & O_ACCMODE) != O_RDONLY) { + if((fi->flags & O_ACCMODE) != O_RDONLY && !FCC_REQ(req)->is_writable(ino)) { + // allow write for key nodes fuse_reply_err(req, EACCES); } else if((err = FCC_REQ(req)->open_file(ino, fi)) != 0) { fuse_reply_err(req, err); @@ -144,11 +148,8 @@ static void fs_read(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, stru dbg_default_trace("entering {}.", __func__); FileBytes* pfb = reinterpret_cast(fi->fh); - if(static_cast(off) < pfb->size) { - fuse_reply_buf(req, reinterpret_cast(pfb->bytes + off), min(pfb->size - off, size)); - } else { - fuse_reply_buf(req, nullptr, 0); - } + reply_buf_limited(req, reinterpret_cast(pfb->bytes), pfb->size, off, size); + dbg_default_trace("leaving {}.", __func__); } @@ -159,6 +160,21 @@ static void fs_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi dbg_default_trace("leaving {}.", __func__); } +static void fs_write(fuse_req_t req, fuse_ino_t ino, const char* buf, size_t size, off_t off, + struct fuse_file_info* fi) { + dbg_default_trace("entering {}.", __func__); + + // TODO write_buf instead?... use fi?? + ssize_t res = FCC_REQ(req)->write_file(ino, buf, size, off, fi); + if(res < 0) { + fuse_reply_err(req, -res); + } else { + fuse_reply_write(req, (size_t)res); + } + + dbg_default_trace("leaving {}.", __func__); +} + static const struct fuse_lowlevel_ops fs_ops = { .init = fs_init, .destroy = fs_destroy, @@ -176,8 +192,7 @@ static const struct fuse_lowlevel_ops fs_ops = { .link = NULL, .open = fs_open, .read = fs_read, - .write = NULL, - .flush = NULL, + .write = fs_write, .release = fs_release, .fsync = NULL, .opendir = NULL, @@ -192,59 +207,42 @@ static const struct fuse_lowlevel_ops fs_ops = { * appear to work fine under the debugger. To avoid the problem, either (a) use absolute pathnames, or (b) record * your current working directory by calling get_current_dir_name before you invoke fuse_main, and then convert * relative pathnames into corresponding absolute ones. Obviously, (b) is the preferred approach. ", - * we need to prepare the configuration file path to make sure the \aServerClient is able to access the configuration + * we need to prepare the configuration file path to make sure the \a ServerClient is able to access the configuration * file. */ void prepare_derecho_conf_file() { - char cwd[4096]; -#pragma GCC diagnostic ignored "-Wunused-result" - getcwd(cwd, 4096); -#pragma GCC diagnostic pop - sprintf(cwd + strlen(cwd), "/derecho.cfg"); - setenv("DERECHO_CONF_FILE", cwd, false); - dbg_default_debug("Using derecho config file:{}.", getenv("DERECHO_CONF_FILE")); -} - -void log_current_dir(bool foreground) { - char buff[FILENAME_MAX]; // create string buffer to hold path - auto working = GetCurrentDir(buff, FILENAME_MAX); - if(working) { - std::string current_working_dir(buff); - - current_working_dir = (foreground) ? current_working_dir + " (running in foreground)" : current_working_dir + " (running in background)"; - auto derecho_conf_file = getenv("DERECHO_CONF_FILE"); - - std::ofstream out("/home/yy354/fuse_log.txt"); - out << current_working_dir; - out << " derecho conf file: [" << derecho_conf_file << "]"; - out.close(); - } -} - -struct fuse_session* se; - -void signalHandler(int signum) { - fuse_session_unmount(se); - free(se); - exit(signum); + std::filesystem::path p = std::filesystem::current_path() / "derecho.cfg"; + const std::string conf_file = p.u8string(); + + const char* const derecho_conf_file = "DERECHO_CONF_FILE"; + setenv(derecho_conf_file, conf_file.c_str(), false); + dbg_default_debug("Using derecho config file: {}.", getenv(derecho_conf_file)); + + // char cwd[4096]; + // #pragma GCC diagnostic ignored "-Wunused-result" + // getcwd(cwd, 4096); + // #pragma GCC diagnostic pop + // sprintf(cwd + strlen(cwd), "/derecho.cfg"); + // setenv("DERECHO_CONF_FILE", cwd, false); + // dbg_default_debug("Using derecho config file:{}.", getenv("DERECHO_CONF_FILE")); } int main(int argc, char** argv) { prepare_derecho_conf_file(); struct fuse_args args = FUSE_ARGS_INIT(argc, argv); - se = nullptr; + struct fuse_session* se = nullptr; struct fuse_cmdline_opts opts; int ret = -1; if(fuse_parse_cmdline(&args, &opts) != 0) { return ret; } + try { - // register signal SIGINT and signal handler - signal(SIGINT, signalHandler); if(opts.show_help) { - std::cout << "usage: " << argv[0] << " [options] " << std::endl; + std::cout << "usage: " << argv[0] << " [options] \n" + << std::endl; fuse_cmdline_help(); fuse_lowlevel_help(); ret = 0; @@ -257,14 +255,27 @@ int main(int argc, char** argv) { } if(opts.mountpoint == nullptr) { - std::cout << "usage: " << argv[0] << " [options] " << std::endl; + std::cout << "usage: " << argv[0] << " [options] \n" + << " " << argv[0] << " --help" << std::endl; ret = 1; throw 1; } - fuse_daemonize(opts.foreground); // start session + // TODO fcc hangs forever when no server nodes running + // fcc also causes signal handlers to not register in fuse_set_signal_handlers + // problem: ServiceClientAPI::get_service_client() registers SIGINT and SIGTERM handlers + // (proof in logging??: [trace] Polling thread ending.) + // fuse_set_signal_handlers only sets when replacing SIG_DFL + FuseClientContextType fcc; + if(fuse_client_signals::store_old_signal_handlers() == -1) { + dbg_default_error("could not store old signal handlers"); + ret = 1; + throw 1; + } + + dbg_default_info("start session"); se = fuse_session_new(&args, &fs_ops, sizeof(fs_ops), &fcc); if(se == nullptr) { throw 1; @@ -276,16 +287,21 @@ int main(int argc, char** argv) { throw 3; } - // log_current_dir(opts.foreground); + fuse_daemonize(opts.foreground); /* Block until ctrl+c or fusermount -u */ + dbg_default_info("starting fuse client."); if(opts.singlethread) { ret = fuse_session_loop(se); } else { - ret = fuse_session_loop_mt(se, opts.clone_fd); + std::cout << "Multi-threaded client not supported yet" << std::endl; + ret = 1; + // ret = fuse_session_loop_mt(se, opts.clone_fd); } + dbg_default_info("ending fuse."); fuse_session_unmount(se); + throw 3; } catch(int& ex) { switch(ex) { case 3: @@ -293,6 +309,10 @@ int main(int argc, char** argv) { case 2: fuse_session_destroy(se); case 1: + if(fuse_client_signals::restore_old_signal_handlers() == -1) { + dbg_default_error("could not restore old signal handlers"); + ret = 1; + } free(opts.mountpoint); fuse_opt_free_args(&args); } diff --git a/src/service/fuse/fuse_client_context.hpp b/src/service/fuse/fuse_client_context.hpp index 0328468b..f2b4b4c0 100644 --- a/src/service/fuse/fuse_client_context.hpp +++ b/src/service/fuse/fuse_client_context.hpp @@ -1,19 +1,25 @@ #pragma once -#include "cascade/object_pool_metadata.hpp" -#include + +#define FUSE_USE_VERSION 31 + +#include #include #include #include +#include +#include + +#include +#include +#include +#include #include #include #include -#include -#include #include #include #include -#define GetCurrentDir getcwd namespace derecho { namespace cascade { @@ -88,6 +94,15 @@ class FuseClientINode { return 0; } + virtual uint64_t write_file(const char* buf, size_t size, off_t off, struct fuse_file_info* fi) { + (void)buf; + (void)size; + (void)off; + (void)fi; + + return -1; + } + virtual void initialize() { } @@ -104,15 +119,62 @@ class FuseClientINode { } } + virtual void write_helper(const char* buf, size_t size, off_t off, + struct fuse_file_info* fi, FileBytes* file_bytes) { + bool appending = false; + + if(fi) { + std::stringstream ss; + ss << std::oct << fi->flags; + std::string flag_str = ss.str(); + + dbg_default_info("file flag: {}", flag_str); + if(fi->flags & O_LARGEFILE) { // TODO + dbg_default_info("large file"); + } + if(fi->flags & O_APPEND) { + dbg_default_info("appending"); + appending = true; + } else { + dbg_default_info("not appending, resizing TODO ?? "); + } + if(fi->flags & O_TRUNC) { + dbg_default_info("truncating"); + } + } + size_t next_size = std::max(file_bytes->size, off + size); + + if(!appending) { + next_size = off + size; // remove extra old content + } + + dbg_default_info("writing. off: {}, buf_size: {}, next_size: {}.", off, size, next_size); + dbg_default_info("writing content: {}.", std::quoted(reinterpret_cast(buf))); + FileBytes next_bytes(next_size); + memcpy(next_bytes.bytes, file_bytes->bytes, next_size); + memcpy(next_bytes.bytes + off, buf, size); + + dbg_default_info("next file: {}.", std::quoted(reinterpret_cast(next_bytes.bytes))); + write_contents(next_bytes.bytes, next_size); + + return; + } + private: // Helper functions for check_update() virtual void update_contents() { return; } + + virtual void write_contents(const uint8_t* content, size_t size) { + return; + } }; template -struct TypeName { static const char* name; }; +struct TypeName { + static const char* name; +}; template const char* TypeName::name = "unknown"; @@ -355,9 +417,8 @@ class ShardMetaINode : public FuseClientINode { } public: - ShardMetaINode(const uint32_t _shard_index, const uint32_t _subgroup_index, ServiceClientAPI& _capi) : shard_index(_shard_index), - subgroup_index(_subgroup_index), - capi(_capi) { + ShardMetaINode(const uint32_t _shard_index, const uint32_t _subgroup_index, ServiceClientAPI& _capi) + : shard_index(_shard_index), subgroup_index(_subgroup_index), capi(_capi) { this->update_interval = 2; this->last_update_sec = 0; this->type = INodeType::META; @@ -389,9 +450,8 @@ class KeyINode : public FuseClientINode { persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. ServiceClientAPI& capi; - KeyINode(typename CascadeType::KeyType& k, fuse_ino_t pino, ServiceClientAPI& _capi) : key(k), - file_bytes(std::make_unique()), - capi(_capi) { + KeyINode(typename CascadeType::KeyType& k, fuse_ino_t pino, ServiceClientAPI& _capi) + : key(k), file_bytes(std::make_unique()), capi(_capi) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); this->update_interval = 2; this->last_update_sec = 0; @@ -420,6 +480,18 @@ class KeyINode : public FuseClientINode { return 0; } + virtual uint64_t write_file(const char* buf, size_t size, off_t off, struct fuse_file_info* fi) override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + + check_update(); + + write_helper(buf, size, off, fi, this->file_bytes.get()); + + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + + return 0; + } + virtual uint64_t get_file_size() override { check_update(); return this->file_bytes.get()->size; @@ -475,6 +547,30 @@ class KeyINode : public FuseClientINode { return; } } + + virtual void write_contents(const uint8_t* content, size_t size) override { + ShardINode* pino_shard = reinterpret_cast*>(this->parent); + SubgroupINode* pino_subgroup = reinterpret_cast*>(pino_shard->parent); + + typename CascadeType::ObjectType obj; + obj.key = this->key; + // TODO + obj.previous_version = INVALID_VERSION; // this->previous_version; + obj.previous_version_by_key = INVALID_VERSION; // this->previous_version_by_key; + obj.blob = Blob(content, size); + + dbg_default_debug("raw put"); + derecho::rpc::QueryResults> result + = capi.template put(obj, pino_subgroup->subgroup_index, pino_shard->shard_index); + + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + dbg_default_debug("node({}) replied with version:{},ts_us:{}", + reply_future.first, std::get<0>(reply), std::get<1>(reply)); + } + + update_contents(); // TODO race conditions ... ?? + } }; class ObjectPoolMetaINode : public FuseClientINode { @@ -550,8 +646,8 @@ class ObjectPoolMetaINode : public FuseClientINode { } public: - ObjectPoolMetaINode(const std::string _cur_pathname, ServiceClientAPI& _capi) : cur_pathname(_cur_pathname), - capi(_capi) { + ObjectPoolMetaINode(const std::string _cur_pathname, ServiceClientAPI& _capi) + : cur_pathname(_cur_pathname), capi(_capi) { this->update_interval = 2; this->last_update_sec = 0; this->type = INodeType::META; @@ -598,8 +694,8 @@ class ObjectPoolPathINode : public FuseClientINode { this->children.emplace_back(std::make_unique(cur_pathname, capi)); } - ObjectPoolPathINode(std::string _cur_pathname, fuse_ino_t pino, ServiceClientAPI& _capi) : capi(_capi), - cur_pathname(_cur_pathname) { + ObjectPoolPathINode(std::string _cur_pathname, fuse_ino_t pino, ServiceClientAPI& _capi) + : capi(_capi), cur_pathname(_cur_pathname) { this->update_interval = 10; this->last_update_sec = 10; this->type = INodeType::OBJECTPOOL_PATH; @@ -784,9 +880,8 @@ class ObjectPoolKeyINode : public FuseClientINode { persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. ServiceClientAPI& capi; - ObjectPoolKeyINode(std::string k, fuse_ino_t pino, ServiceClientAPI& _capi) : key(k), - file_bytes(std::make_unique()), - capi(_capi) { + ObjectPoolKeyINode(std::string k, fuse_ino_t pino, ServiceClientAPI& _capi) + : key(k), file_bytes(std::make_unique()), capi(_capi) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); this->update_interval = 2; this->last_update_sec = 0; @@ -807,6 +902,18 @@ class ObjectPoolKeyINode : public FuseClientINode { return 0; } + virtual uint64_t write_file(const char* buf, size_t size, off_t off, struct fuse_file_info* fi) override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + + check_update(); + + write_helper(buf, size, off, fi, this->file_bytes.get()); + + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + + return 0; + } + virtual uint64_t get_file_size() override { dbg_default_debug("----GET FILE SIZE key is [{}].", this->key); check_update(); @@ -839,6 +946,25 @@ class ObjectPoolKeyINode : public FuseClientINode { } dbg_default_trace("\n \n ----OBJP keyInode update content [{}] leaving {}.", gettid(), __func__); } + + virtual void write_contents(const uint8_t* content, size_t size) override { + ObjectWithStringKey obj; + obj.key = this->key; + obj.previous_version = INVALID_VERSION; + obj.previous_version_by_key = INVALID_VERSION; + obj.blob = Blob(content, size); + + dbg_default_debug("object pool put"); + derecho::rpc::QueryResults> result = capi.put(obj); + + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + dbg_default_debug("node({}) replied with version:{},ts_us:{}", + reply_future.first, std::get<0>(reply), std::get<1>(reply)); + } + + update_contents(); // TODO race conditions ... ?? + } }; class MetadataServiceRootINode : public FuseClientINode { @@ -888,7 +1014,8 @@ class DLLINode : public FuseClientINode { public: std::string file_name; // DLL id? ServiceClientAPI& capi; - DLLINode(std::string& _filename, fuse_ino_t pino, ServiceClientAPI& _capi) : file_name(_filename), capi(_capi) { + DLLINode(std::string& _filename, fuse_ino_t pino, ServiceClientAPI& _capi) + : file_name(_filename), capi(_capi) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); this->type = INodeType::DLL; this->display_name = std::string("dllfile") + _filename; @@ -924,7 +1051,8 @@ class DLLINode : public FuseClientINode { }; /** - * The fuse filesystem context for fuse_client. This context will be used as 'userdata' on starting a fuse session. + * The fuse filesystem context for fuse_client. + * This context will be used as 'userdata' on starting a fuse session. */ template @@ -981,7 +1109,7 @@ class FuseClientContext { admin_metadata_inode(capi) {} /** initialize */ void initialize(const json& group_layout) { - dbg_default_trace("[{}]entering {} .", gettid(), __func__); + dbg_default_trace("[{}]entering {}.", gettid(), __func__); populate_inodes(group_layout); this->admin_metadata_inode.initialize(); clock_gettime(CLOCK_REALTIME, &this->init_timestamp); @@ -1063,7 +1191,7 @@ class FuseClientContext { stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; break; case INodeType::KEY: - stbuf.st_mode = S_IFREG | 0444; + stbuf.st_mode = S_IFREG | 0744; // TODO set 0444 for unimplemented stbuf.st_size = pfci->get_file_size(); stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; @@ -1099,6 +1227,28 @@ class FuseClientContext { return timeout_sec; } + // TODO docs + bool is_writable(fuse_ino_t ino) { + FuseClientINode* pfci = reinterpret_cast(ino); + // TODO only allow for object pool?? (or not) + return pfci->type == INodeType::KEY; + } + + int write_file(fuse_ino_t ino, const char* buf, size_t size, off_t off, struct fuse_file_info* fi) { + dbg_default_trace("[{}]entering {} with ino={:x}.", gettid(), __func__, ino); + + if(!is_writable(ino)) { + return -EACCES; + } + FuseClientINode* pfci = reinterpret_cast(ino); + pfci->write_file(buf, size, off, fi); + // TODO errors?? + ssize_t result = size; + + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return result; + } + /** * open a file. * @param ino inode diff --git a/src/service/fuse/fuse_client_signals.cpp b/src/service/fuse/fuse_client_signals.cpp new file mode 100644 index 00000000..5f68fd44 --- /dev/null +++ b/src/service/fuse/fuse_client_signals.cpp @@ -0,0 +1,49 @@ +#include "fuse_client_signals.hpp" + +#include +#include +#include + +using handler_t = void (*)(int); +static std::map old_signal_handlers; +static const int signals[] = {SIGHUP, SIGINT, SIGTERM, SIGPIPE}; + +static int set_one_signal_handler(int sig, int remove) { + if(remove) { + struct sigaction old_sa; + if(sigaction(sig, NULL, &old_sa) == -1) { + return -1; + } + old_signal_handlers[sig] = old_sa.sa_handler; + } + + struct sigaction sa; + memset(&sa, 0, sizeof(struct sigaction)); + bool contains = old_signal_handlers.find(sig) == old_signal_handlers.end(); + sa.sa_handler = remove || !contains ? SIG_DFL : old_signal_handlers[sig]; + sigemptyset(&(sa.sa_mask)); + sa.sa_flags = 0; + if(sigaction(sig, &sa, NULL) == -1) { + return -1; + } + + return 0; +} + +int fuse_client_signals::store_old_signal_handlers() { + for(int sig : signals) { + if(set_one_signal_handler(sig, true) == -1) { + return -1; + } + } + return 0; +} + +int fuse_client_signals::restore_old_signal_handlers() { + for(int sig : signals) { + if(set_one_signal_handler(sig, false) == -1) { + return -1; + } + } + return 0; +} diff --git a/src/service/fuse/fuse_client_signals.hpp b/src/service/fuse/fuse_client_signals.hpp new file mode 100644 index 00000000..366588eb --- /dev/null +++ b/src/service/fuse/fuse_client_signals.hpp @@ -0,0 +1,7 @@ +#pragma once + +namespace fuse_client_signals { +int store_old_signal_handlers(); + +int restore_old_signal_handlers(); +} // namespace fuse_client_signals From 3caa5567f5d703ee6e000f483106b0a25b06855e Mon Sep 17 00:00:00 2001 From: Michael Ye Date: Thu, 12 Jan 2023 15:59:33 -0500 Subject: [PATCH 05/10] feat: add proper truncate and file create in op --- .../{cascade_runner => cascade_runner.sh} | 3 +- src/service/client.cpp | 2240 ++++++++--------- src/service/fuse/fuse_client.cpp | 186 +- src/service/fuse/fuse_client_context.hpp | 394 +-- 4 files changed, 1438 insertions(+), 1385 deletions(-) rename scripts/util/{cascade_runner => cascade_runner.sh} (98%) diff --git a/scripts/util/cascade_runner b/scripts/util/cascade_runner.sh similarity index 98% rename from scripts/util/cascade_runner rename to scripts/util/cascade_runner.sh index 8aca3924..7c57c4fa 100755 --- a/scripts/util/cascade_runner +++ b/scripts/util/cascade_runner.sh @@ -2,7 +2,6 @@ set -euo pipefail -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") SCRIPT_NAME=$(basename "${BASH_SOURCE[0]}") HELP_STRING="A helper script to set up a local cascade instance Usage: $SCRIPT_NAME [options] @@ -21,7 +20,7 @@ Usage: $SCRIPT_NAME [options] -F D directory to mount (runs in foreground). -c I Start client node at folder cfg/nI." -CFG_DIR="$SCRIPT_DIR/cfg" +CFG_DIR="$PWD/cfg" FUSE_FOREGROUND=false FUSE_DIR="" HELP=false diff --git a/src/service/client.cpp b/src/service/client.cpp index a87442bf..e437e701 100644 --- a/src/service/client.cpp +++ b/src/service/client.cpp @@ -1,20 +1,20 @@ #include -#include -#include +#include #include -#include -#include -#include +#include #include -#include +#include +#include +#include #include +#include #ifdef ENABLE_EVALUATION #include "perftest.hpp" -#endif//ENABLE_EVALUATION +#endif // ENABLE_EVALUATION using namespace derecho::cascade; -#define PROC_NAME "cascade_client" +#define PROC_NAME "cascade_client" template void print_subgroup_member(ServiceClientAPI& capi, uint32_t subgroup_index) { @@ -22,9 +22,9 @@ void print_subgroup_member(ServiceClientAPI& capi, uint32_t subgroup_index) { << "subgroup_index=" << subgroup_index << ")" << std::endl; auto members = capi.template get_subgroup_members(subgroup_index); uint32_t shard_index = 0; - for (const auto& shard: members) { + for(const auto& shard : members) { std::cout << "shard-" << shard_index << " = ["; - for (const auto& nid: shard) { + for(const auto& nid : shard) { std::cout << nid << ","; } std::cout << "]" << std::endl; @@ -35,9 +35,9 @@ void print_subgroup_member(ServiceClientAPI& capi, const std::string& op) { std::cout << "Object Pool=" << op << std::endl; auto members = capi.get_subgroup_members(op); uint32_t shard_index = 0; - for (const auto& shard: members) { + for(const auto& shard : members) { std::cout << "shard-" << shard_index << " = ["; - for (const auto& nid: shard) { + for(const auto& nid : shard) { std::cout << nid << ","; } std::cout << "]" << std::endl; @@ -48,9 +48,9 @@ template void print_shard_member(ServiceClientAPI& capi, uint32_t subgroup_index, uint32_t shard_index) { std::cout << "Subgroup (Type=" << std::type_index(typeid(SubgroupType)).name() << "," << "subgroup_index=" << subgroup_index << "," - << "shard_index=" << shard_index << ") member list = ["; - auto members = capi.template get_shard_members(subgroup_index,shard_index); - for (auto nid : members) { + << "shard_index=" << shard_index << ") member list = ["; + auto members = capi.template get_shard_members(subgroup_index, shard_index); + for(auto nid : members) { std::cout << nid << ","; } std::cout << "]" << std::endl; @@ -59,14 +59,13 @@ void print_shard_member(ServiceClientAPI& capi, uint32_t subgroup_index, uint32_ void print_shard_member(ServiceClientAPI& capi, const std::string& op, uint32_t shard_index) { std::cout << "Object Pool=" << op << ",\n" << "shard_index=" << shard_index << ",\nmember list=["; - auto members = capi.get_shard_members(op,shard_index); - for (auto nid : members) { + auto members = capi.get_shard_members(op, shard_index); + for(auto nid : members) { std::cout << nid << ","; } std::cout << "]" << std::endl; } - /** disabled void print_shard_member(ServiceClientAPI& capi, derecho::subgroup_id_t subgroup_id, uint32_t shard_index) { std::cout << "subgroup_id=" << subgroup_id << "," @@ -80,31 +79,30 @@ void print_shard_member(ServiceClientAPI& capi, derecho::subgroup_id_t subgroup_ **/ /** - * IMPORTANT: the order of the policy_name has to match ShardMemberSelectionPolicy + * IMPORTANT: the order of the policy_name has to match ShardMemberSelectionPolicy * defined in include/cascade/service.hpp */ static const char* policy_names[] = { - "FirstMember", - "LastMember", - "Random", - "FixedRandom", - "RoundRobin", - "KeyHashing", - "UserSpecified", - nullptr -}; + "FirstMember", + "LastMember", + "Random", + "FixedRandom", + "RoundRobin", + "KeyHashing", + "UserSpecified", + nullptr}; inline ShardMemberSelectionPolicy parse_policy_name(const std::string& policy_name) { ShardMemberSelectionPolicy policy = ShardMemberSelectionPolicy::FirstMember; - int i=1; - while(policy_names[i]){ - if (policy_name == policy_names[i]) { + int i = 1; + while(policy_names[i]) { + if(policy_name == policy_names[i]) { policy = static_cast(i); break; } i++; } - if (policy_names[i] == nullptr) { + if(policy_names[i] == nullptr) { return ShardMemberSelectionPolicy::InvalidPolicy; } return policy; @@ -114,15 +112,15 @@ template void print_member_selection_policy(ServiceClientAPI& capi, uint32_t subgroup_index, uint32_t shard_index) { std::cout << "Subgroup (Type=" << std::type_index(typeid(SubgroupType)).name() << "," << "subgroup_index=" << subgroup_index << "," - << "shard_index=" << shard_index << ") policy="; - auto policy = capi.template get_member_selection_policy(subgroup_index,shard_index); + << "shard_index=" << shard_index << ") policy="; + auto policy = capi.template get_member_selection_policy(subgroup_index, shard_index); std::cout << policy_names[std::get<0>(policy)] << "(" << std::get<0>(policy) << ")," << std::get<1>(policy) << "" << std::endl; } template void set_member_selection_policy(ServiceClientAPI& capi, uint32_t subgroup_index, uint32_t shard_index, - ShardMemberSelectionPolicy policy, node_id_t user_specified_node_id) { - capi.template set_member_selection_policy(subgroup_index,shard_index,policy,user_specified_node_id); + ShardMemberSelectionPolicy policy, node_id_t user_specified_node_id) { + capi.template set_member_selection_policy(subgroup_index, shard_index, policy, user_specified_node_id); } /* TEST1: members */ @@ -130,25 +128,25 @@ void member_test(ServiceClientAPI& capi) { // print all members. std::cout << "Top Derecho group members = ["; auto members = capi.get_members(); - for (auto nid: members) { - std::cout << nid << "," ; + for(auto nid : members) { + std::cout << nid << ","; } std::cout << "]" << std::endl; // print per Subgroup Members: - print_shard_member(capi,0,0); - print_shard_member(capi,0,0); + print_shard_member(capi, 0, 0); + print_shard_member(capi, 0, 0); } -static std::vector tokenize(std::string &line, const char *delimiter) { +static std::vector tokenize(std::string& line, const char* delimiter) { std::vector tokens; char line_buf[1024]; std::strcpy(line_buf, line.c_str()); - char *token = std::strtok(line_buf, delimiter); - while (token != nullptr) { + char* token = std::strtok(line_buf, delimiter); + while(token != nullptr) { tokens.push_back(std::string(token)); token = std::strtok(NULL, delimiter); } - return tokens; // RVO + return tokens; // RVO } static void print_red(std::string msg) { @@ -163,30 +161,30 @@ static void print_cyan(std::string msg) { << "\033[0m" << std::endl; } -#define on_subgroup_type(x, ft, ...) \ - if ((x) == "VCSS") { \ - ft (__VA_ARGS__); \ - } else if ((x) == "PCSS") { \ - ft (__VA_ARGS__); \ - } else if ((x) == "TCSS") { \ - ft (__VA_ARGS__); \ - } else { \ - print_red("unknown subgroup type:" + x); \ +#define on_subgroup_type(x, ft, ...) \ + if((x) == "VCSS") { \ + ft(__VA_ARGS__); \ + } else if((x) == "PCSS") { \ + ft(__VA_ARGS__); \ + } else if((x) == "TCSS") { \ + ft(__VA_ARGS__); \ + } else { \ + print_red("unknown subgroup type:" + x); \ } -#define check_put_and_remove_result(result) \ - for (auto& reply_future:result.get()) {\ - auto reply = reply_future.second.get();\ - std::cout << "node(" << reply_future.first << ") replied with version:" << std::get<0>(reply)\ - << ",ts_us:" << std::get<1>(reply) << std::endl;\ +#define check_put_and_remove_result(result) \ + for(auto& reply_future : result.get()) { \ + auto reply = reply_future.second.get(); \ + std::cout << "node(" << reply_future.first << ") replied with version:" << std::get<0>(reply) \ + << ",ts_us:" << std::get<1>(reply) << std::endl; \ } template void put(ServiceClientAPI& capi, const std::string& key, const std::string& value, persistent::version_t pver, persistent::version_t pver_bk, uint32_t subgroup_index, uint32_t shard_index) { typename SubgroupType::ObjectType obj; - if constexpr (std::is_same::value) { - obj.key = static_cast(std::stol(key,nullptr,0)); - } else if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { + obj.key = static_cast(std::stol(key, nullptr, 0)); + } else if constexpr(std::is_same::value) { obj.key = key; } else { print_red(std::string("Unhandled KeyType:") + typeid(typename SubgroupType::KeyType).name()); @@ -194,17 +192,17 @@ void put(ServiceClientAPI& capi, const std::string& key, const std::string& valu } obj.previous_version = pver; obj.previous_version_by_key = pver_bk; - obj.blob = Blob(reinterpret_cast(value.c_str()),value.length()); - derecho::rpc::QueryResults> result = capi.template put(obj, subgroup_index, shard_index); + obj.blob = Blob(reinterpret_cast(value.c_str()), value.length()); + derecho::rpc::QueryResults> result = capi.template put(obj, subgroup_index, shard_index); check_put_and_remove_result(result); } template void put_and_forget(ServiceClientAPI& capi, const std::string& key, const std::string& value, persistent::version_t pver, persistent::version_t pver_bk, uint32_t subgroup_index, uint32_t shard_index) { typename SubgroupType::ObjectType obj; - if constexpr (std::is_same::value) { - obj.key = static_cast(std::stol(key,nullptr,0)); - } else if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { + obj.key = static_cast(std::stol(key, nullptr, 0)); + } else if constexpr(std::is_same::value) { obj.key = key; } else { print_red(std::string("Unhandled KeyType:") + typeid(typename SubgroupType::KeyType).name()); @@ -212,7 +210,7 @@ void put_and_forget(ServiceClientAPI& capi, const std::string& key, const std::s } obj.previous_version = pver; obj.previous_version_by_key = pver_bk; - obj.blob = Blob(reinterpret_cast(value.c_str()),value.length()); + obj.blob = Blob(reinterpret_cast(value.c_str()), value.length()); capi.template put_and_forget(obj, subgroup_index, shard_index); std::cout << "put done." << std::endl; } @@ -222,14 +220,14 @@ void op_put(ServiceClientAPI& capi, const std::string& key, const std::string& v obj.key = key; obj.previous_version = pver; obj.previous_version_by_key = pver_bk; - obj.blob = Blob(reinterpret_cast(value.c_str()),value.length()); - derecho::rpc::QueryResults> result = capi.put(obj); + obj.blob = Blob(reinterpret_cast(value.c_str()), value.length()); + derecho::rpc::QueryResults> result = capi.put(obj); check_put_and_remove_result(result); } void op_put_file(ServiceClientAPI& capi, const std::string& key, const std::string& filename, persistent::version_t pver, persistent::version_t pver_bk) { // get file size - std::ifstream value_file(filename,std::ios::binary); + std::ifstream value_file(filename, std::ios::binary); if(!value_file.good()) { dbg_default_error("Cannot open file:{} for read.", filename); throw std::runtime_error("Cannot open file:" + filename + "for read"); @@ -238,14 +236,14 @@ void op_put_file(ServiceClientAPI& capi, const std::string& key, const std::stri std::size_t file_size = value_file.tellg(); value_file.seekg(0); // message generator - blob_generator_func_t message_generator = [&value_file] (uint8_t* buffer, const std::size_t size) { - value_file.read(reinterpret_cast(buffer),size); + blob_generator_func_t message_generator = [&value_file](uint8_t* buffer, const std::size_t size) { + value_file.read(reinterpret_cast(buffer), size); return size; }; - ObjectWithStringKey obj(key,message_generator,file_size); + ObjectWithStringKey obj(key, message_generator, file_size); obj.previous_version = pver; obj.previous_version_by_key = pver_bk; - derecho::rpc::QueryResults> result = capi.put(obj); + derecho::rpc::QueryResults> result = capi.put(obj); value_file.close(); check_put_and_remove_result(result); } @@ -255,14 +253,14 @@ void op_put_and_forget(ServiceClientAPI& capi, const std::string& key, const std obj.key = key; obj.previous_version = pver; obj.previous_version_by_key = pver_bk; - obj.blob = Blob(reinterpret_cast(value.c_str()),value.length()); + obj.blob = Blob(reinterpret_cast(value.c_str()), value.length()); capi.put_and_forget(obj); std::cout << "put done." << std::endl; } void op_put_file_and_forget(ServiceClientAPI& capi, const std::string& key, const std::string& filename, persistent::version_t pver, persistent::version_t pver_bk) { // get file size - std::ifstream value_file(filename,std::ios::binary); + std::ifstream value_file(filename, std::ios::binary); if(!value_file.good()) { dbg_default_error("Cannot open file:{} for read.", filename); throw std::runtime_error("Cannot open file:" + filename + "for read"); @@ -271,11 +269,11 @@ void op_put_file_and_forget(ServiceClientAPI& capi, const std::string& key, cons std::size_t file_size = value_file.tellg(); value_file.seekg(0); // message generator - blob_generator_func_t message_generator = [&value_file] (uint8_t* buffer, const std::size_t size) { - value_file.read(reinterpret_cast(buffer),size); + blob_generator_func_t message_generator = [&value_file](uint8_t* buffer, const std::size_t size) { + value_file.read(reinterpret_cast(buffer), size); return size; }; - ObjectWithStringKey obj(key,message_generator,file_size); + ObjectWithStringKey obj(key, message_generator, file_size); obj.previous_version = pver; obj.previous_version_by_key = pver_bk; capi.put_and_forget(obj); @@ -284,27 +282,27 @@ void op_put_file_and_forget(ServiceClientAPI& capi, const std::string& key, cons template void create_object_pool(ServiceClientAPI& capi, const std::string& id, uint32_t subgroup_index) { - auto result = capi.template create_object_pool(id,subgroup_index); + auto result = capi.template create_object_pool(id, subgroup_index); check_put_and_remove_result(result); std::cout << "create_object_pool is done." << std::endl; -} +} template void trigger_put(ServiceClientAPI& capi, const std::string& key, const std::string& value, uint32_t subgroup_index, uint32_t shard_index) { typename SubgroupType::ObjectType obj; - if constexpr (std::is_same::value) { - obj.key = static_cast(std::stol(key,nullptr,0)); - } else if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { + obj.key = static_cast(std::stol(key, nullptr, 0)); + } else if constexpr(std::is_same::value) { obj.key = key; } else { print_red(std::string("Unhandled KeyType:") + typeid(typename SubgroupType::KeyType).name()); return; } - obj.blob = Blob(reinterpret_cast(value.c_str()),value.length()); + obj.blob = Blob(reinterpret_cast(value.c_str()), value.length()); derecho::rpc::QueryResults result = capi.template trigger_put(obj, subgroup_index, shard_index); result.get(); - + std::cout << "trigger_put is done." << std::endl; } @@ -312,47 +310,46 @@ void op_trigger_put(ServiceClientAPI& capi, const std::string& key, const std::s ObjectWithStringKey obj; obj.key = key; - obj.blob = Blob(reinterpret_cast(value.c_str()),value.length()); + obj.blob = Blob(reinterpret_cast(value.c_str()), value.length()); derecho::rpc::QueryResults result = capi.trigger_put(obj); result.get(); - + std::cout << "op_trigger_put is done." << std::endl; } template void collective_trigger_put(ServiceClientAPI& capi, const std::string& key, const std::string& value, uint32_t subgroup_index, std::vector nodes) { typename SubgroupType::ObjectType obj; - if constexpr (std::is_same::value) { - obj.key = static_cast(std::stol(key,nullptr,0)); - } else if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { + obj.key = static_cast(std::stol(key, nullptr, 0)); + } else if constexpr(std::is_same::value) { obj.key = key; } else { print_red(std::string("Unhandled KeyType:") + typeid(typename SubgroupType::KeyType).name()); return; } - obj.blob = Blob(reinterpret_cast(value.c_str()),value.length()); - std::unordered_map>> nodes_and_futures; - for (auto& nid: nodes) { - nodes_and_futures.emplace(nid,nullptr); + obj.blob = Blob(reinterpret_cast(value.c_str()), value.length()); + std::unordered_map>> nodes_and_futures; + for(auto& nid : nodes) { + nodes_and_futures.emplace(nid, nullptr); } capi.template collective_trigger_put(obj, subgroup_index, nodes_and_futures); - for (auto& kv: nodes_and_futures) { + for(auto& kv : nodes_and_futures) { kv.second.get(); std::cout << "Finish sending to node " << kv.first << std::endl; } - - std::cout << "collective_trigger_put is done." << std::endl; + std::cout << "collective_trigger_put is done." << std::endl; } template void remove(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index) { - if constexpr (std::is_same::value) { - derecho::rpc::QueryResults> result = std::move(capi.template remove(static_cast(std::stol(key,nullptr,0)), subgroup_index, shard_index)); + if constexpr(std::is_same::value) { + derecho::rpc::QueryResults> result = std::move(capi.template remove(static_cast(std::stol(key, nullptr, 0)), subgroup_index, shard_index)); check_put_and_remove_result(result); - } else if constexpr (std::is_same::value) { - derecho::rpc::QueryResults> result = std::move(capi.template remove(key, subgroup_index, shard_index)); + } else if constexpr(std::is_same::value) { + derecho::rpc::QueryResults> result = std::move(capi.template remove(key, subgroup_index, shard_index)); check_put_and_remove_result(result); } else { print_red(std::string("Unhandled KeyType:") + typeid(typename SubgroupType::KeyType).name()); @@ -365,120 +362,120 @@ void op_remove(ServiceClientAPI& capi, const std::string& key) { check_put_and_remove_result(result); } -#define check_get_result(result) \ - for (auto& reply_future:result.get()) {\ - auto reply = reply_future.second.get();\ - std::cout << "node(" << reply_future.first << ") replied with value:" << reply << std::endl;\ +#define check_get_result(result) \ + for(auto& reply_future : result.get()) { \ + auto reply = reply_future.second.get(); \ + std::cout << "node(" << reply_future.first << ") replied with value:" << reply << std::endl; \ } template -void get(ServiceClientAPI& capi, const std::string& key, persistent::version_t ver, bool stable, uint32_t subgroup_index,uint32_t shard_index) { - if constexpr (std::is_same::value) { +void get(ServiceClientAPI& capi, const std::string& key, persistent::version_t ver, bool stable, uint32_t subgroup_index, uint32_t shard_index) { + if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get( - static_cast(std::stol(key,nullptr,0)),ver,stable,subgroup_index,shard_index); + static_cast(std::stol(key, nullptr, 0)), ver, stable, subgroup_index, shard_index); check_get_result(result); - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get( - key,ver,stable,subgroup_index,shard_index); + key, ver, stable, subgroup_index, shard_index); check_get_result(result); } } template void get_by_time(ServiceClientAPI& capi, const std::string& key, uint64_t ts_us, bool stable, uint32_t subgroup_index, uint32_t shard_index) { - if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get_by_time( - static_cast(std::stol(key,nullptr,0)),ts_us,stable,subgroup_index,shard_index); + static_cast(std::stol(key, nullptr, 0)), ts_us, stable, subgroup_index, shard_index); check_get_result(result); - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get_by_time( - key,ts_us,stable,subgroup_index,shard_index); + key, ts_us, stable, subgroup_index, shard_index); check_get_result(result); } } template void multi_get(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index) { - if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template multi_get( - static_cast(std::stol(key,nullptr,0)),subgroup_index,shard_index); + static_cast(std::stol(key, nullptr, 0)), subgroup_index, shard_index); check_get_result(result); - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template multi_get( - key,subgroup_index,shard_index); + key, subgroup_index, shard_index); check_get_result(result); } } template -void get_size(ServiceClientAPI& capi, const std::string& key, persistent::version_t ver, bool stable, uint32_t subgroup_index,uint32_t shard_index) { - if constexpr (std::is_same::value) { +void get_size(ServiceClientAPI& capi, const std::string& key, persistent::version_t ver, bool stable, uint32_t subgroup_index, uint32_t shard_index) { + if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get_size( - static_cast(std::stol(key,nullptr,0)),ver,stable,subgroup_index,shard_index); + static_cast(std::stol(key, nullptr, 0)), ver, stable, subgroup_index, shard_index); check_get_result(result); - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get_size( - key,ver,stable,subgroup_index,shard_index); + key, ver, stable, subgroup_index, shard_index); check_get_result(result); } } template void multi_get_size(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index) { - if constexpr ( std::is_same::value) { - derecho::rpc::QueryResults result = capi.template multi_get_size ( - static_cast(std::stol(key,nullptr,0)),subgroup_index,shard_index); + if constexpr(std::is_same::value) { + derecho::rpc::QueryResults result = capi.template multi_get_size( + static_cast(std::stol(key, nullptr, 0)), subgroup_index, shard_index); check_get_result(result); - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template multi_get_size( - key,subgroup_index,shard_index); + key, subgroup_index, shard_index); check_get_result(result); } } template void get_size_by_time(ServiceClientAPI& capi, const std::string& key, uint64_t ts_us, bool stable, uint32_t subgroup_index, uint32_t shard_index) { - if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get_size_by_time( - static_cast(std::stol(key,nullptr,0)),ts_us,stable,subgroup_index,shard_index); + static_cast(std::stol(key, nullptr, 0)), ts_us, stable, subgroup_index, shard_index); check_get_result(result); - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get_size_by_time( - key,ts_us,stable,subgroup_index,shard_index); + key, ts_us, stable, subgroup_index, shard_index); check_get_result(result); } } -#define check_list_keys_result(result) \ - for (auto& reply_future:result.get()) {\ - auto reply = reply_future.second.get();\ - std::cout << "Keys:" << std::endl;\ - for (auto& key:reply) {\ - std::cout << " " << key << std::endl;\ - }\ +#define check_list_keys_result(result) \ + for(auto& reply_future : result.get()) { \ + auto reply = reply_future.second.get(); \ + std::cout << "Keys:" << std::endl; \ + for(auto& key : reply) { \ + std::cout << " " << key << std::endl; \ + } \ } -#define check_op_list_keys_result(result)\ - std::cout << "Keys:" << std::endl;\ - for (auto& key:result) {\ - std::cout << " " << key << std::endl;\ +#define check_op_list_keys_result(result) \ + std::cout << "Keys:" << std::endl; \ + for(auto& key : result) { \ + std::cout << " " << key << std::endl; \ } template void multi_list_keys(ServiceClientAPI& capi, uint32_t subgroup_index, uint32_t shard_index) { - derecho::rpc::QueryResults> result = capi.template multi_list_keys(subgroup_index,shard_index); + derecho::rpc::QueryResults> result = capi.template multi_list_keys(subgroup_index, shard_index); check_list_keys_result(result); } template void list_keys(ServiceClientAPI& capi, persistent::version_t ver, bool stable, uint32_t subgroup_index, uint32_t shard_index) { - derecho::rpc::QueryResults> result = capi.template list_keys(ver,stable,subgroup_index,shard_index); + derecho::rpc::QueryResults> result = capi.template list_keys(ver, stable, subgroup_index, shard_index); check_list_keys_result(result); } template void list_keys_by_time(ServiceClientAPI& capi, uint64_t ts_us, bool stable, uint32_t subgroup_index, uint32_t shard_index) { - derecho::rpc::QueryResults> result = capi.template list_keys_by_time(ts_us,stable,subgroup_index,shard_index); + derecho::rpc::QueryResults> result = capi.template list_keys_by_time(ts_us, stable, subgroup_index, shard_index); check_list_keys_result(result); } @@ -487,13 +484,14 @@ void list_keys_by_time(ServiceClientAPI& capi, uint64_t ts_us, bool stable, uint template void list_data_by_prefix(ServiceClientAPI& capi, std::string prefix, persistent::version_t ver, uint32_t subgroup_index, uint32_t shard_index) { std::vector keys; - for (auto& obj : from_shard(keys,capi,subgroup_index,shard_index,ver).where([&prefix](typename SubgroupType::ObjectType o){ - if (o.blob.size < prefix.size()) { - return false; - } else { - return (std::string(reinterpret_cast(o.blob.bytes),prefix.size()) == prefix); - } - }).toStdVector()) { + for(auto& obj : from_shard(keys, capi, subgroup_index, shard_index, ver).where([&prefix](typename SubgroupType::ObjectType o) { + if(o.blob.size < prefix.size()) { + return false; + } else { + return (std::string(reinterpret_cast(o.blob.bytes), prefix.size()) == prefix); + } + }) + .toStdVector()) { std::cout << "Found:" << obj << std::endl; } } @@ -503,87 +501,90 @@ void list_data_by_prefix(ServiceClientAPI& c print_red("TCSS does not support list_data_by_prefix."); } - // "list_data_between_versions [version_begin] [version_end]\n\t test LINQ api - version_iterator \n" template -void list_data_between_versions(ServiceClientAPI &capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, persistent::version_t ver_begin, persistent::version_t ver_end) { - if constexpr (std::is_same::value) { - auto result = capi.template get(static_cast(std::stol(key,nullptr,0)), ver_end, subgroup_index, shard_index); - for (auto &reply_future : result.get()) { +void list_data_between_versions(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, persistent::version_t ver_begin, persistent::version_t ver_end) { + if constexpr(std::is_same::value) { + auto result = capi.template get(static_cast(std::stol(key, nullptr, 0)), ver_end, subgroup_index, shard_index); + for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); - if (reply.is_valid()) { + if(reply.is_valid()) { ver_end = reply.version; } else { return; } } - for (auto &obj : from_versions(static_cast(std::stol(key,nullptr,0)), capi, subgroup_index, shard_index, ver_end).where([ver_begin](typename SubgroupType::ObjectType obj) { - return ver_begin == INVALID_VERSION || obj.version >= ver_begin; - }).toStdVector()) { + for(auto& obj : from_versions(static_cast(std::stol(key, nullptr, 0)), capi, subgroup_index, shard_index, ver_end).where([ver_begin](typename SubgroupType::ObjectType obj) { + return ver_begin == INVALID_VERSION || obj.version >= ver_begin; + }) + .toStdVector()) { std::cout << "Found:" << obj << std::endl; } - } else if constexpr (std::is_same::value) { - auto result = capi.template get(key, ver_end, true/*always use stable here*/, subgroup_index, shard_index); - for (auto &reply_future : result.get()) { + } else if constexpr(std::is_same::value) { + auto result = capi.template get(key, ver_end, true /*always use stable here*/, subgroup_index, shard_index); + for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); - if (reply.is_valid()) { + if(reply.is_valid()) { ver_end = reply.version; } else { return; } } - for (auto &obj : from_versions(key, capi, subgroup_index, shard_index, ver_end).where([ver_begin](typename SubgroupType::ObjectType obj) { - return ver_begin == INVALID_VERSION || obj.version >= ver_begin; - }).toStdVector()) { + for(auto& obj : from_versions(key, capi, subgroup_index, shard_index, ver_end).where([ver_begin](typename SubgroupType::ObjectType obj) { + return ver_begin == INVALID_VERSION || obj.version >= ver_begin; + }) + .toStdVector()) { std::cout << "Found:" << obj << std::endl; } } } template <> -void list_data_between_versions(ServiceClientAPI &capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, persistent::version_t ver_begin, persistent::version_t ver_end) { +void list_data_between_versions(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, persistent::version_t ver_begin, persistent::version_t ver_end) { print_red("TCSS does not support list_data_between_versions."); } template -void list_data_between_timestamps(ServiceClientAPI &capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, uint64_t ts_begin, uint64_t ts_end) { +void list_data_between_timestamps(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, uint64_t ts_begin, uint64_t ts_end) { std::vector keys; - if constexpr (std::is_same::value) { - auto result = capi.template get(static_cast(std::stol(key,nullptr,0)), CURRENT_VERSION, subgroup_index, shard_index); - for (auto &reply_future : result.get()) { + if constexpr(std::is_same::value) { + auto result = capi.template get(static_cast(std::stol(key, nullptr, 0)), CURRENT_VERSION, subgroup_index, shard_index); + for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); - if (reply.is_valid()) { + if(reply.is_valid()) { ts_end = reply.timestamp_us >= ts_end ? ts_end : reply.timestamp_us; } else { return; } } - for (auto &obj : from_shard_by_time(keys, capi, subgroup_index, shard_index, ts_end).where([&key,ts_begin](typename SubgroupType::ObjectType obj) { - return !obj.is_null() && static_cast(std::stol(key,nullptr,0)) == obj.key && obj.timestamp_us >= ts_begin; - }).toStdVector()) { + for(auto& obj : from_shard_by_time(keys, capi, subgroup_index, shard_index, ts_end).where([&key, ts_begin](typename SubgroupType::ObjectType obj) { + return !obj.is_null() && static_cast(std::stol(key, nullptr, 0)) == obj.key && obj.timestamp_us >= ts_begin; + }) + .toStdVector()) { std::cout << "Found:" << obj << std::endl; } - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { // set the timestamp to the latest update if ts_end > latest_ts - auto result = capi.template get(key, CURRENT_VERSION, true/*always stable version here*/, subgroup_index, shard_index); - for (auto &reply_future : result.get()) { + auto result = capi.template get(key, CURRENT_VERSION, true /*always stable version here*/, subgroup_index, shard_index); + for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); - if (reply.is_valid()) { + if(reply.is_valid()) { ts_end = reply.timestamp_us >= ts_end ? ts_end : reply.timestamp_us; } else { return; } } - for (auto &obj : from_shard_by_time(keys, capi, subgroup_index, shard_index, ts_end).where([&key,ts_begin](typename SubgroupType::ObjectType obj) { - return (!obj.is_null() && key == obj.key && obj.timestamp_us >= ts_begin); - }).toStdVector()) { + for(auto& obj : from_shard_by_time(keys, capi, subgroup_index, shard_index, ts_end).where([&key, ts_begin](typename SubgroupType::ObjectType obj) { + return (!obj.is_null() && key == obj.key && obj.timestamp_us >= ts_begin); + }) + .toStdVector()) { std::cout << "Found:" << obj << std::endl; } } } template <> -void list_data_between_timestamps(ServiceClientAPI &capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, uint64_t ts_begin, uint64_t ts_end) { +void list_data_between_timestamps(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, uint64_t ts_begin, uint64_t ts_end) { print_red("TCSS does not support list_data_between_timestamp."); } @@ -593,9 +594,9 @@ void list_data_in_subgroup(ServiceClientAPI& capi, uint32_t subgroup_index, pers std::vector keys; std::vector> shard_linq_list; - std::unordered_map> shardidx_to_keys; + std::unordered_map> shardidx_to_keys; - for (auto &obj : from_subgroup(shardidx_to_keys, shard_linq_list, capi, subgroup_index, version).toStdVector()) { + for(auto& obj : from_subgroup(shardidx_to_keys, shard_linq_list, capi, subgroup_index, version).toStdVector()) { std::cout << "Found:" << obj << std::endl; } } @@ -608,19 +609,19 @@ void list_data_in_subgroup(ServiceClientAPI& template void list_data_in_objectpool(ServiceClientAPI& capi, persistent::version_t version, const std::string& objpool_path) { std::vector keys; - for (auto &obj : from_objectpool(capi,keys,version,objpool_path).toStdVector()) { + for(auto& obj : from_objectpool(capi, keys, version, objpool_path).toStdVector()) { std::cout << "Found:" << obj << std::endl; } } -#endif// HAS_BOOLINQ +#endif // HAS_BOOLINQ // register notification template bool register_notification(ServiceClientAPI& capi, uint32_t subgroup_index) { return capi.template register_notification_handler( - [](const Blob& msg)->void{ + [](const Blob& msg) -> void { std::cout << "Subgroup Notification received:" - << "data:" << std::string(reinterpret_cast(msg.bytes),msg.size) + << "data:" << std::string(reinterpret_cast(msg.bytes), msg.size) << std::endl; }, subgroup_index); @@ -629,7 +630,7 @@ bool register_notification(ServiceClientAPI& capi, uint32_t subgroup_index) { // unregister notification template bool unregister_notification(ServiceClientAPI& capi, uint32_t subgroup_index) { - return capi.template register_notification_handler({},subgroup_index); + return capi.template register_notification_handler({}, subgroup_index); } #ifdef ENABLE_EVALUATION @@ -644,8 +645,8 @@ bool perftest(PerfTestClient& ptc, uint64_t duration_secs, const std::string& output_file) { debug_enter_func_with_args("put_type={},object_pool_pathname={},ec2cs={},read_write_ratio={},ops_threshold={},duration_secs={},output_file={}", - put_type,object_pool_pathname,static_cast(ec2cs),read_write_ratio,ops_threshold,duration_secs,output_file); - bool ret = ptc.template perf_put(put_type,object_pool_pathname,ec2cs,read_write_ratio,ops_threshold,duration_secs,output_file); + put_type, object_pool_pathname, static_cast(ec2cs), read_write_ratio, ops_threshold, duration_secs, output_file); + bool ret = ptc.template perf_put(put_type, object_pool_pathname, ec2cs, read_write_ratio, ops_threshold, duration_secs, output_file); debug_leave_func(); return ret; } @@ -662,64 +663,63 @@ bool perftest(PerfTestClient& ptc, uint64_t duration_secs, const std::string& output_file) { debug_enter_func_with_args("put_type={},subgroup_index={},shard_index={},ec2cs={},read_write_ratio={},ops_threshold={},duration_secs={},output_file={}", - put_type,subgroup_index, shard_index,static_cast(ec2cs),read_write_ratio,ops_threshold,duration_secs,output_file); - bool ret = ptc.template perf_put(put_type,subgroup_index,shard_index,ec2cs,read_write_ratio,ops_threshold,duration_secs,output_file); + put_type, subgroup_index, shard_index, static_cast(ec2cs), read_write_ratio, ops_threshold, duration_secs, output_file); + bool ret = ptc.template perf_put(put_type, subgroup_index, shard_index, ec2cs, read_write_ratio, ops_threshold, duration_secs, output_file); debug_leave_func(); return ret; } template -bool perftest_ordered_put(ServiceClientAPI &capi, +bool perftest_ordered_put(ServiceClientAPI& capi, uint32_t message_size, uint64_t duration_sec, uint32_t subgroup_index, - uint32_t shard_index){ + uint32_t shard_index) { debug_enter_func_with_args("message_size={},duration_sec={},subgroup_index={},shard_index={}.", - message_size,duration_sec,subgroup_index,shard_index); - auto result = capi.template perf_put(message_size,duration_sec,subgroup_index,shard_index); + message_size, duration_sec, subgroup_index, shard_index); + auto result = capi.template perf_put(message_size, duration_sec, subgroup_index, shard_index); check_get_result(result); debug_leave_func(); return true; } template <> -bool perftest_ordered_put(ServiceClientAPI &capi, - uint32_t message_size, - uint64_t duration_sec, - uint32_t subgroup_index, - uint32_t shard_index){ +bool perftest_ordered_put(ServiceClientAPI& capi, + uint32_t message_size, + uint64_t duration_sec, + uint32_t subgroup_index, + uint32_t shard_index) { print_red("TCSS does not support perftest_ordered_put"); return false; } template -bool dump_timestamp(ServiceClientAPI &capi, +bool dump_timestamp(ServiceClientAPI& capi, uint32_t subgroup_index, uint32_t shard_index, const std::string& filename) { debug_enter_func_with_args("subgroup_index={}, shard_index={}, filename={}", - subgroup_index,shard_index,filename); - auto result = capi.template dump_timestamp(filename,subgroup_index,shard_index); + subgroup_index, shard_index, filename); + auto result = capi.template dump_timestamp(filename, subgroup_index, shard_index); result.get(); TimestampLogger::flush(filename); debug_leave_func(); return true; } -#endif // ENABLE_EVALUATION - +#endif // ENABLE_EVALUATION /* TEST2: put/get/remove tests */ -using command_handler_t = std::function& cmd_tokens)>; +using command_handler_t = std::function& cmd_tokens)>; struct command_entry_t { - const std::string cmd; // command name - const std::string desc; // help info - const std::string help; // full help - const command_handler_t handler; // handler + const std::string cmd; // command name + const std::string desc; // help info + const std::string help; // full help + const command_handler_t handler; // handler }; void list_commands(const std::vector& command_list) { - for (const auto& entry: command_list) { - if (entry.handler) { + for(const auto& entry : command_list) { + if(entry.handler) { std::cout << std::left << std::setw(32) << entry.cmd << "- " << entry.desc << std::endl; } else { print_cyan("# " + entry.cmd + " #"); @@ -729,974 +729,850 @@ void list_commands(const std::vector& command_list) { ssize_t find_command(const std::vector& command_list, const std::string& command) { ssize_t pos = 0; - for(;pos < static_cast(command_list.size());pos++) { - if (command_list.at(pos).cmd == command) { + for(; pos < static_cast(command_list.size()); pos++) { + if(command_list.at(pos).cmd == command) { break; } } - if (pos == static_cast(command_list.size())) { + if(pos == static_cast(command_list.size())) { pos = -1; } return pos; } - bool shell_is_active = true; #define SUBGROUP_TYPE_LIST "VCSS|PCSS|TCSS" #define SHARD_MEMBER_SELECTION_POLICY_LIST "FirstMember|LastMember|Random|FixedRandom|RoundRobin|KeyHashing|UserSpecified" -#define CHECK_FORMAT(tks,argc) \ - if (tks.size() < argc) { \ - print_red("Invalid command format. Please try help " + tks[0] + "."); \ - return false; \ - } -std::vector commands = -{ - { - "General Commands","","",command_handler_t() - }, - { - "help", - "Print help info", - "help [command name]", - [](ServiceClientAPI&,const std::vector& cmd_tokens){ - if (cmd_tokens.size() >= 2) { - ssize_t command_index = find_command(commands,cmd_tokens[1]); - if (command_index < 0) { - print_red("unknown command:'"+cmd_tokens[1]+"'."); - } else { - std::cout << commands.at(command_index).help << std::endl; - } - return (command_index>=0); - } else { - list_commands(commands); - return true; - } - } - }, - { - "quit", - "Exit", - "quit", - [](ServiceClientAPI&,const std::vector& cmd_tokens) { - shell_is_active = false; - return true; - } - }, - { - "Membership Commands","","",command_handler_t() - }, - { - "list_members", - "List the IDs of all nodes in the Cascade service.", - "list_members", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - std::cout << "Cascade service members = ["; - auto members = capi.get_members(); - for (auto nid: members) { - std::cout << nid << "," ; - } - std::cout << "]" << std::endl; - return true; - } - }, - { - "list_subgroup_members", - "List the nodes in a subgroup specified by type and subgroup index.", - "list_subgroup_members [subgroup index(default:0)]\n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - uint32_t subgroup_index = 0; - CHECK_FORMAT(cmd_tokens,2); - if (cmd_tokens.size() >= 3) { - subgroup_index = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - } - on_subgroup_type(cmd_tokens[1],print_subgroup_member,capi,subgroup_index); - return true; - } - }, - { - "op_list_subgroup_members", - "List the subgroup members by object pool name.", - "op_list_subgroup_members ", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,2); - print_subgroup_member(capi,cmd_tokens[1]); - return true; - } - }, - { - "list_shard_members", - "List the IDs in a shard specified by type, subgroup index, and shard index.", - "list_shard_members [subgroup index(default:0)] [shard index(default:0)]\n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - uint32_t subgroup_index = 0, shard_index = 0; - CHECK_FORMAT(cmd_tokens,2); - if (cmd_tokens.size() >= 3) { - subgroup_index = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - } - if (cmd_tokens.size() >= 4) { - shard_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - } - on_subgroup_type(cmd_tokens[1],print_shard_member,capi,subgroup_index,shard_index); - return true; - } - }, - { - "op_list_shard_members", - "List the shard members by object pool name.", - "op_list_shard_members [shard index(default:0)]", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - uint32_t shard_index = 0; - CHECK_FORMAT(cmd_tokens,2); - if (cmd_tokens.size() >= 3) { - shard_index = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - } - print_shard_member(capi,cmd_tokens[1],shard_index); - return true; - } - }, - { - "set_member_selection_policy", - "Set the policy for choosing among a set of server members.", - "set_member_selection_policy [user specified node id]\n" - "type := " SUBGROUP_TYPE_LIST "\n" - "policy := " SHARD_MEMBER_SELECTION_POLICY_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,5); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - ShardMemberSelectionPolicy policy = parse_policy_name(cmd_tokens[4]); - if (policy == ShardMemberSelectionPolicy::InvalidPolicy) { - print_red("Invalid policy name:" + cmd_tokens[4]); - return false; - } - node_id_t user_specified_node_id = INVALID_NODE_ID; - if (cmd_tokens.size() >= 6) { - user_specified_node_id = static_cast(std::stoi(cmd_tokens[5],nullptr,0)); - } - on_subgroup_type(cmd_tokens[1],set_member_selection_policy,capi,subgroup_index,shard_index,policy,user_specified_node_id); - return true; - } - }, - { - "get_member_selection_policy", - "Get the policy for choosing among a set of server members.", - "get_member_selection_policy \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,4); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - on_subgroup_type(cmd_tokens[1],print_member_selection_policy,capi,subgroup_index,shard_index); - return true; - } - }, - { - "Object Pool Manipulation Commands","","",command_handler_t() - }, - { - "list_object_pools", - "List existing object pools", - "list_object_pools", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - std::cout << "refreshed object pools:" << std::endl; - for (std::string& opath: capi.list_object_pools(true)) { - std::cout << "\t" << opath << std::endl; - } - return true; - } - }, - { - "create_object_pool", - "Create an object pool", - "create_object_pool \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,4); - std::string opath = cmd_tokens[1]; - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - on_subgroup_type(cmd_tokens[2],create_object_pool,capi,opath,subgroup_index); - return true; - } - }, - { - "remove_object_pool", - "Soft-Remove an object pool", - "remove_object_pool ", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,2); - auto result = capi.remove_object_pool(cmd_tokens[1]); - check_put_and_remove_result(result); - return true; - } - }, - { - "get_object_pool", - "Get details of an object pool", - "get_object_pool ", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,2); - auto opm = capi.find_object_pool(cmd_tokens[1]); - std::cout << "get_object_pool returns:" - << opm << std::endl; - return true; - } - }, - { - "Object Maniputlation Commands","","",command_handler_t() - }, - { - "put", - "Put an object to a shard.", - "put [previous_version(default:-1)] [previous_version_by_key(default:-1)]\n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - persistent::version_t pver = persistent::INVALID_VERSION; - persistent::version_t pver_bk = persistent::INVALID_VERSION; - CHECK_FORMAT(cmd_tokens,6); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[5],nullptr,0)); - if (cmd_tokens.size() >= 7) - pver = static_cast(std::stol(cmd_tokens[6],nullptr,0)); - if (cmd_tokens.size() >= 8) - pver_bk = static_cast(std::stol(cmd_tokens[7],nullptr,0)); - on_subgroup_type(cmd_tokens[1],put,capi,cmd_tokens[2]/*key*/,cmd_tokens[3]/*value*/,pver,pver_bk,subgroup_index,shard_index); - return true; - } - }, - { - "put_and_forget", - "Put an object to a shard, without a return value", - "put_and_forget [previous_version(default:-1)] [previous_version_by_key(default:-1)]\n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - persistent::version_t pver = persistent::INVALID_VERSION; - persistent::version_t pver_bk = persistent::INVALID_VERSION; - CHECK_FORMAT(cmd_tokens,6); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[5],nullptr,0)); - if (cmd_tokens.size() >= 7) - pver = static_cast(std::stol(cmd_tokens[6],nullptr,0)); - if (cmd_tokens.size() >= 8) - pver_bk = static_cast(std::stol(cmd_tokens[7],nullptr,0)); - on_subgroup_type(cmd_tokens[1],put_and_forget,capi,cmd_tokens[2]/*key*/,cmd_tokens[3]/*value*/,pver,pver_bk,subgroup_index,shard_index); - return true; - } - }, - { - "op_put", - "Put an object into an object pool", - "op_put [previous_version(default:-1)] [previous_version_by_key(default:-1)]\n" - "Please note that cascade automatically decides the object pool path using the key's prefix.", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - persistent::version_t pver = persistent::INVALID_VERSION; - persistent::version_t pver_bk = persistent::INVALID_VERSION; - CHECK_FORMAT(cmd_tokens,3); - if (cmd_tokens.size() >= 4) - pver = static_cast(std::stol(cmd_tokens[3],nullptr,0)); - if (cmd_tokens.size() >= 5) - pver_bk = static_cast(std::stol(cmd_tokens[4],nullptr,0)); - op_put(capi,cmd_tokens[1]/*key*/,cmd_tokens[2]/*value*/,pver,pver_bk); - return true; - } - }, - { - "op_put_file", - "Put an object into an object pool, where object's value is from a file,", - "op_put_file [previous_version(default:-1)] [previous_version_by_key(default:-1)]\n" - "Please note that cascade automatically decides the object pool path using the key's prefix.", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - persistent::version_t pver = persistent::INVALID_VERSION; - persistent::version_t pver_bk = persistent::INVALID_VERSION; - CHECK_FORMAT(cmd_tokens,3); - if (cmd_tokens.size() >= 4) - pver = static_cast(std::stol(cmd_tokens[3],nullptr,0)); - if (cmd_tokens.size() >= 5) - pver_bk = static_cast(std::stol(cmd_tokens[4],nullptr,0)); - op_put_file(capi,cmd_tokens[1]/*key*/,cmd_tokens[2]/*filename*/,pver,pver_bk); - return true; - } - }, - { - "op_put_and_forget", - "Put an object into an object pool, without a return value", - "op_put_and_forget [previous_version(default:-1)] [previous_version_by_key(default:-1)]\n" - "type := " SUBGROUP_TYPE_LIST "\n" - "Please note that cascade automatically decides the object pool path using the key's prefix.", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - persistent::version_t pver = persistent::INVALID_VERSION; - persistent::version_t pver_bk = persistent::INVALID_VERSION; - CHECK_FORMAT(cmd_tokens,3); - if (cmd_tokens.size() >= 4) - pver = static_cast(std::stol(cmd_tokens[3],nullptr,0)); - if (cmd_tokens.size() >= 5) - pver_bk = static_cast(std::stol(cmd_tokens[4],nullptr,0)); - op_put_and_forget(capi,cmd_tokens[1]/*key*/,cmd_tokens[2]/*value*/,pver,pver_bk); - return true; - } - }, - { - "op_put_file_and_forget", - "Put an object into an object pool, where object's value is from a file,", - "op_put_file_and_forget [previous_version(default:-1)] [previous_version_by_key(default:-1)]\n" - "Please note that cascade automatically decides the object pool path using the key's prefix.", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - persistent::version_t pver = persistent::INVALID_VERSION; - persistent::version_t pver_bk = persistent::INVALID_VERSION; - CHECK_FORMAT(cmd_tokens,3); - if (cmd_tokens.size() >= 4) - pver = static_cast(std::stol(cmd_tokens[3],nullptr,0)); - if (cmd_tokens.size() >= 5) - pver_bk = static_cast(std::stol(cmd_tokens[4],nullptr,0)); - op_put_file_and_forget(capi,cmd_tokens[1]/*key*/,cmd_tokens[2]/*filename*/,pver,pver_bk); - return true; - } - }, - { - "trigger_put", - "Trigger put an object to a shard.", - "trigger_put \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,6); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[5],nullptr,0)); - on_subgroup_type(cmd_tokens[1],trigger_put,capi,cmd_tokens[2]/*key*/,cmd_tokens[3]/*value*/,subgroup_index,shard_index); - return true; - } - }, - { - "op_trigger_put", - "Trigger put an object to an object pool.", - "op_trigger_put \n" - "Please note that cascade automatically decides the object pool path using the key's prefix.", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,3); - op_trigger_put(capi,cmd_tokens[1]/*key*/,cmd_tokens[2]/*value*/); - return true; - } - }, - { - "collective_trigger_put", - "Collectively trigger put an object to a set of nodes in a subgroup.", - "collective_trigger_put [node id 2, ...] \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - std::vector nodes; - CHECK_FORMAT(cmd_tokens,6); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - size_t arg_idx = 5; - while(arg_idx < cmd_tokens.size()) { - nodes.push_back(static_cast(std::stoi(cmd_tokens[arg_idx++],nullptr,0))); - } - on_subgroup_type(cmd_tokens[1],collective_trigger_put,capi,cmd_tokens[2]/*key*/,cmd_tokens[3]/*value*/,subgroup_index,nodes); - return true; - } - }, - { - "remove", - "Remove an object from a shard.", - "remove \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,5); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - on_subgroup_type(cmd_tokens[1],remove,capi,cmd_tokens[2]/*key*/,subgroup_index,shard_index); - return true; - } - }, - { - "op_remove", - "Remove an object from an object pool.", - "op_remove \n" - "Please note that cascade automatically decides the object pool path using the key's prefix.", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,2); - op_remove(capi,cmd_tokens[1]); - return true; - } - }, - { - "get", - "Get an object (by version).", - "get [ version(default:current version) ]\n" - "type := " SUBGROUP_TYPE_LIST "\n" - "stable := 0|1 using stable data or not.", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,6); - bool stable = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[5],nullptr,0)); - persistent::version_t version = CURRENT_VERSION; - if (cmd_tokens.size() >= 7) { - version = static_cast(std::stol(cmd_tokens[6],nullptr,0)); - } - on_subgroup_type(cmd_tokens[1],get,capi,cmd_tokens[2],version,stable,subgroup_index,shard_index); - return true; - } - }, - { - "op_get", - "Get an object from an object pool (by version).", - "op_get [ version(default:current version) ]\n" - "stable := 0|1 using stable data or not.\n" - "Please note that cascade automatically decides the object pool path using the key's prefix.", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,3); - bool stable = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - persistent::version_t version = CURRENT_VERSION; - if (cmd_tokens.size() >= 4) { - version = static_cast(std::stol(cmd_tokens[3],nullptr,0)); - } - auto res = capi.get(cmd_tokens[1],version,stable); - check_get_result(res); - return true; - } - }, - { - "get_by_time", - "Get an object (by timestamp in microseconds).", - "get_by_time \n" - "type := " SUBGROUP_TYPE_LIST "\n" - "stable := 0|1 using stable data or not", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,7); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - uint64_t ts_us = static_cast(std::stol(cmd_tokens[5],nullptr,0)); - bool stable = static_cast(std::stoi(cmd_tokens[6],nullptr,0)); - on_subgroup_type(cmd_tokens[1],get_by_time,capi,cmd_tokens[2],ts_us,stable,subgroup_index,shard_index); - return true; - } - }, - { - "op_get_by_time", - "Get an object from an object pool (by timestamp in microseconds).", - "op_get_by_time \n" - "stable := 0|1 using stable data or not\n" - "Please note that cascade automatically decides the object pool path using the key's prefix.", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,4); - uint64_t ts_us = static_cast(std::stol(cmd_tokens[2],nullptr,0)); - bool stable = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - auto res = capi.get_by_time(cmd_tokens[1],ts_us,stable); - check_get_result(res); - return true; - } - }, - { - "multi_get", - "Get an object, which will participate atomic broadcast for the latest value.", - "multi_get \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,5); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - on_subgroup_type(cmd_tokens[1],multi_get,capi,cmd_tokens[2],subgroup_index,shard_index); - return true; - } - }, - { - "op_multi_get", - "Get an object, which will participate atomic broadcast for the latest value.", - "op_multi_get \n", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,2); - auto res = capi.multi_get(cmd_tokens[1]); - check_get_result(res); - return true; - } - }, - { - "multi_get_size", - "Get the size of an object, which will participate atomic broadcast for the latest size.", - "multi_get_size \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,5); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - on_subgroup_type(cmd_tokens[1],multi_get_size,capi,cmd_tokens[2],subgroup_index,shard_index); - return true; - } - }, - { - "op_multi_get_size", - "Get the size of an object, which will participate atomic broadcast for the latest size.", - "op_multi_get_size \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,2); - auto res = capi.multi_get_size(cmd_tokens[1]); - check_get_result(res); - return true; - } - }, - { - "get_size", - "Get the size of an object (by version).", - "get_size [ version(default:current version) ]\n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,6); - bool stable = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[5],nullptr,0)); - persistent::version_t version = CURRENT_VERSION; - if (cmd_tokens.size() >= 7) { - version = static_cast(std::stol(cmd_tokens[6],nullptr,0)); - } - on_subgroup_type(cmd_tokens[1],get_size,capi,cmd_tokens[2],version,stable,subgroup_index,shard_index); - return true; - } - }, - { - "op_get_size", - "Get the size of an object from an object pool (by version).", - "op_get_size [ version(default:current version) ]\n" - "Please note that cascade automatically decides the object pool path using the key's prefix.", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,3); - persistent::version_t version = CURRENT_VERSION; - bool stable = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - if (cmd_tokens.size() >= 4) { - version = static_cast(std::stol(cmd_tokens[3],nullptr,0)); - } - auto res = capi.get_size(cmd_tokens[1],version,stable); - check_get_result(res); - return true; - } - }, - { - "get_size_by_time", - "Get the size of an object (by timestamp in microseconds).", - "get_size_by_time \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,7); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - uint64_t ts_us = static_cast(std::stol(cmd_tokens[5],nullptr,0)); - bool stable = static_cast(std::stoi(cmd_tokens[6],nullptr,0)); - on_subgroup_type(cmd_tokens[1],get_size_by_time,capi,cmd_tokens[2],ts_us,stable,subgroup_index,shard_index); - return true; - } - }, - { - "op_get_size_by_time", - "Get the size of an object from an object pool (by timestamp in microseconds).", - "op_get_size_by_time \n" - "Please note that cascade automatically decides the object pool path using the key's prefix.", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,4); - uint64_t ts_us = static_cast(std::stol(cmd_tokens[2],nullptr,0)); - bool stable = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - auto res = capi.get_size_by_time(cmd_tokens[1],ts_us,stable); - check_get_result(res); - return true; - } - }, - { - "multi_list_keys", - "list the object keys in a shard using atomic broadcast for the latest version.", - "multi_list_keys \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,4); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - on_subgroup_type(cmd_tokens[1],multi_list_keys,capi,subgroup_index,shard_index); - return true; - } - }, - { - "op_multi_list_keys", - "list the object keys in a shard using atomic broadcast for the latest version.", - "op_multi_list_keys \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,2); - auto result = capi.multi_list_keys(cmd_tokens[1]); - check_op_list_keys_result(capi.wait_list_keys(result)); - return true; - } - }, - { - "list_keys", - "list the object keys in a shard (by version).", - "list_keys [ version(default:current version) ]\n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,5); - bool stable = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - persistent::version_t version = CURRENT_VERSION; - if (cmd_tokens.size() >= 6) { - version = static_cast(std::stol(cmd_tokens[5],nullptr,0)); - } - on_subgroup_type(cmd_tokens[1],list_keys,capi,version,stable,subgroup_index,shard_index); - return true; - } - }, - { - "op_list_keys", - "list the object keys in an object pool (by version).", - "op_list_keys [ version(default:current version) ]\n", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,3); - bool stable = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - persistent::version_t version = CURRENT_VERSION; - if (cmd_tokens.size() >= 4) { - version = static_cast(std::stol(cmd_tokens[3],nullptr,0)); - } - auto result = capi.list_keys(version,stable,cmd_tokens[1]); - check_op_list_keys_result(capi.wait_list_keys(result)); - return true; - } - }, - { - "list_keys_by_time", - "list the object keys in a shard (by timestamp in mircoseconds).", - "list_keys_by_time \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,6); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - uint64_t ts_us = static_cast(std::stoull(cmd_tokens[4],nullptr,0)); - bool stable = static_cast(std::stoi(cmd_tokens[5],nullptr,0)); - on_subgroup_type(cmd_tokens[1],list_keys_by_time,capi,ts_us,stable,subgroup_index,shard_index); - return true; - } - }, - { - "op_list_keys_by_time", - "list the object keys in an object pool (by timestamp in microseconds).", - "op_list_keys_by_time \n", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,4); - uint64_t ts_us = static_cast(std::stoull(cmd_tokens[2],nullptr,0)); - bool stable = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - auto result = capi.list_keys_by_time(ts_us,stable,cmd_tokens[1]); - check_op_list_keys_result(capi.wait_list_keys(result)); - return true; - } - }, +#define CHECK_FORMAT(tks, argc) \ + if(tks.size() < argc) { \ + print_red("Invalid command format. Please try help " + tks[0] + "."); \ + return false; \ + } +std::vector commands = { + {"General Commands", "", "", command_handler_t()}, + {"help", + "Print help info", + "help [command name]", + [](ServiceClientAPI&, const std::vector& cmd_tokens) { + if(cmd_tokens.size() >= 2) { + ssize_t command_index = find_command(commands, cmd_tokens[1]); + if(command_index < 0) { + print_red("unknown command:'" + cmd_tokens[1] + "'."); + } else { + std::cout << commands.at(command_index).help << std::endl; + } + return (command_index >= 0); + } else { + list_commands(commands); + return true; + } + }}, + {"quit", + "Exit", + "quit", + [](ServiceClientAPI&, const std::vector& cmd_tokens) { + shell_is_active = false; + return true; + }}, + {"Membership Commands", "", "", command_handler_t()}, + {"list_members", + "List the IDs of all nodes in the Cascade service.", + "list_members", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + std::cout << "Cascade service members = ["; + auto members = capi.get_members(); + for(auto nid : members) { + std::cout << nid << ","; + } + std::cout << "]" << std::endl; + return true; + }}, + {"list_subgroup_members", + "List the nodes in a subgroup specified by type and subgroup index.", + "list_subgroup_members [subgroup index(default:0)]\n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + uint32_t subgroup_index = 0; + CHECK_FORMAT(cmd_tokens, 2); + if(cmd_tokens.size() >= 3) { + subgroup_index = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + } + on_subgroup_type(cmd_tokens[1], print_subgroup_member, capi, subgroup_index); + return true; + }}, + {"op_list_subgroup_members", + "List the subgroup members by object pool name.", + "op_list_subgroup_members ", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 2); + print_subgroup_member(capi, cmd_tokens[1]); + return true; + }}, + {"list_shard_members", + "List the IDs in a shard specified by type, subgroup index, and shard index.", + "list_shard_members [subgroup index(default:0)] [shard index(default:0)]\n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + uint32_t subgroup_index = 0, shard_index = 0; + CHECK_FORMAT(cmd_tokens, 2); + if(cmd_tokens.size() >= 3) { + subgroup_index = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + } + if(cmd_tokens.size() >= 4) { + shard_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + } + on_subgroup_type(cmd_tokens[1], print_shard_member, capi, subgroup_index, shard_index); + return true; + }}, + {"op_list_shard_members", + "List the shard members by object pool name.", + "op_list_shard_members [shard index(default:0)]", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + uint32_t shard_index = 0; + CHECK_FORMAT(cmd_tokens, 2); + if(cmd_tokens.size() >= 3) { + shard_index = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + } + print_shard_member(capi, cmd_tokens[1], shard_index); + return true; + }}, + {"set_member_selection_policy", + "Set the policy for choosing among a set of server members.", + "set_member_selection_policy [user specified node id]\n" + "type := " SUBGROUP_TYPE_LIST "\n" + "policy := " SHARD_MEMBER_SELECTION_POLICY_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 5); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + ShardMemberSelectionPolicy policy = parse_policy_name(cmd_tokens[4]); + if(policy == ShardMemberSelectionPolicy::InvalidPolicy) { + print_red("Invalid policy name:" + cmd_tokens[4]); + return false; + } + node_id_t user_specified_node_id = INVALID_NODE_ID; + if(cmd_tokens.size() >= 6) { + user_specified_node_id = static_cast(std::stoi(cmd_tokens[5], nullptr, 0)); + } + on_subgroup_type(cmd_tokens[1], set_member_selection_policy, capi, subgroup_index, shard_index, policy, user_specified_node_id); + return true; + }}, + {"get_member_selection_policy", + "Get the policy for choosing among a set of server members.", + "get_member_selection_policy \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 4); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + on_subgroup_type(cmd_tokens[1], print_member_selection_policy, capi, subgroup_index, shard_index); + return true; + }}, + {"Object Pool Manipulation Commands", "", "", command_handler_t()}, + {"list_object_pools", + "List existing object pools", + "list_object_pools", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + std::cout << "refreshed object pools:" << std::endl; + for(std::string& opath : capi.list_object_pools(true)) { + std::cout << "\t" << opath << std::endl; + } + return true; + }}, + {"create_object_pool", + "Create an object pool", + "create_object_pool \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 4); + std::string opath = cmd_tokens[1]; + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + on_subgroup_type(cmd_tokens[2], create_object_pool, capi, opath, subgroup_index); + return true; + }}, + {"remove_object_pool", + "Soft-Remove an object pool", + "remove_object_pool ", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 2); + auto result = capi.remove_object_pool(cmd_tokens[1]); + check_put_and_remove_result(result); + return true; + }}, + {"get_object_pool", + "Get details of an object pool", + "get_object_pool ", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 2); + auto opm = capi.find_object_pool(cmd_tokens[1]); + std::cout << "get_object_pool returns:" + << opm << std::endl; + return true; + }}, + {"Object Manipulation Commands", "", "", command_handler_t()}, + {"put", + "Put an object to a shard.", + "put [previous_version(default:-1)] [previous_version_by_key(default:-1)]\n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + persistent::version_t pver = persistent::INVALID_VERSION; + persistent::version_t pver_bk = persistent::INVALID_VERSION; + CHECK_FORMAT(cmd_tokens, 6); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[5], nullptr, 0)); + if(cmd_tokens.size() >= 7) + pver = static_cast(std::stol(cmd_tokens[6], nullptr, 0)); + if(cmd_tokens.size() >= 8) + pver_bk = static_cast(std::stol(cmd_tokens[7], nullptr, 0)); + on_subgroup_type(cmd_tokens[1], put, capi, cmd_tokens[2] /*key*/, cmd_tokens[3] /*value*/, pver, pver_bk, subgroup_index, shard_index); + return true; + }}, + {"put_and_forget", + "Put an object to a shard, without a return value", + "put_and_forget [previous_version(default:-1)] [previous_version_by_key(default:-1)]\n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + persistent::version_t pver = persistent::INVALID_VERSION; + persistent::version_t pver_bk = persistent::INVALID_VERSION; + CHECK_FORMAT(cmd_tokens, 6); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[5], nullptr, 0)); + if(cmd_tokens.size() >= 7) + pver = static_cast(std::stol(cmd_tokens[6], nullptr, 0)); + if(cmd_tokens.size() >= 8) + pver_bk = static_cast(std::stol(cmd_tokens[7], nullptr, 0)); + on_subgroup_type(cmd_tokens[1], put_and_forget, capi, cmd_tokens[2] /*key*/, cmd_tokens[3] /*value*/, pver, pver_bk, subgroup_index, shard_index); + return true; + }}, + {"op_put", + "Put an object into an object pool", + "op_put [previous_version(default:-1)] [previous_version_by_key(default:-1)]\n" + "Please note that cascade automatically decides the object pool path using the key's prefix.", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + persistent::version_t pver = persistent::INVALID_VERSION; + persistent::version_t pver_bk = persistent::INVALID_VERSION; + CHECK_FORMAT(cmd_tokens, 3); + if(cmd_tokens.size() >= 4) + pver = static_cast(std::stol(cmd_tokens[3], nullptr, 0)); + if(cmd_tokens.size() >= 5) + pver_bk = static_cast(std::stol(cmd_tokens[4], nullptr, 0)); + op_put(capi, cmd_tokens[1] /*key*/, cmd_tokens[2] /*value*/, pver, pver_bk); + return true; + }}, + {"op_put_file", + "Put an object into an object pool, where object's value is from a file,", + "op_put_file [previous_version(default:-1)] [previous_version_by_key(default:-1)]\n" + "Please note that cascade automatically decides the object pool path using the key's prefix.", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + persistent::version_t pver = persistent::INVALID_VERSION; + persistent::version_t pver_bk = persistent::INVALID_VERSION; + CHECK_FORMAT(cmd_tokens, 3); + if(cmd_tokens.size() >= 4) + pver = static_cast(std::stol(cmd_tokens[3], nullptr, 0)); + if(cmd_tokens.size() >= 5) + pver_bk = static_cast(std::stol(cmd_tokens[4], nullptr, 0)); + op_put_file(capi, cmd_tokens[1] /*key*/, cmd_tokens[2] /*filename*/, pver, pver_bk); + return true; + }}, + {"op_put_and_forget", + "Put an object into an object pool, without a return value", + "op_put_and_forget [previous_version(default:-1)] [previous_version_by_key(default:-1)]\n" + "type := " SUBGROUP_TYPE_LIST "\n" + "Please note that cascade automatically decides the object pool path using the key's prefix.", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + persistent::version_t pver = persistent::INVALID_VERSION; + persistent::version_t pver_bk = persistent::INVALID_VERSION; + CHECK_FORMAT(cmd_tokens, 3); + if(cmd_tokens.size() >= 4) + pver = static_cast(std::stol(cmd_tokens[3], nullptr, 0)); + if(cmd_tokens.size() >= 5) + pver_bk = static_cast(std::stol(cmd_tokens[4], nullptr, 0)); + op_put_and_forget(capi, cmd_tokens[1] /*key*/, cmd_tokens[2] /*value*/, pver, pver_bk); + return true; + }}, + {"op_put_file_and_forget", + "Put an object into an object pool, where object's value is from a file,", + "op_put_file_and_forget [previous_version(default:-1)] [previous_version_by_key(default:-1)]\n" + "Please note that cascade automatically decides the object pool path using the key's prefix.", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + persistent::version_t pver = persistent::INVALID_VERSION; + persistent::version_t pver_bk = persistent::INVALID_VERSION; + CHECK_FORMAT(cmd_tokens, 3); + if(cmd_tokens.size() >= 4) + pver = static_cast(std::stol(cmd_tokens[3], nullptr, 0)); + if(cmd_tokens.size() >= 5) + pver_bk = static_cast(std::stol(cmd_tokens[4], nullptr, 0)); + op_put_file_and_forget(capi, cmd_tokens[1] /*key*/, cmd_tokens[2] /*filename*/, pver, pver_bk); + return true; + }}, + {"trigger_put", + "Trigger put an object to a shard.", + "trigger_put \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 6); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[5], nullptr, 0)); + on_subgroup_type(cmd_tokens[1], trigger_put, capi, cmd_tokens[2] /*key*/, cmd_tokens[3] /*value*/, subgroup_index, shard_index); + return true; + }}, + {"op_trigger_put", + "Trigger put an object to an object pool.", + "op_trigger_put \n" + "Please note that cascade automatically decides the object pool path using the key's prefix.", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 3); + op_trigger_put(capi, cmd_tokens[1] /*key*/, cmd_tokens[2] /*value*/); + return true; + }}, + {"collective_trigger_put", + "Collectively trigger put an object to a set of nodes in a subgroup.", + "collective_trigger_put [node id 2, ...] \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + std::vector nodes; + CHECK_FORMAT(cmd_tokens, 6); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + size_t arg_idx = 5; + while(arg_idx < cmd_tokens.size()) { + nodes.push_back(static_cast(std::stoi(cmd_tokens[arg_idx++], nullptr, 0))); + } + on_subgroup_type(cmd_tokens[1], collective_trigger_put, capi, cmd_tokens[2] /*key*/, cmd_tokens[3] /*value*/, subgroup_index, nodes); + return true; + }}, + {"remove", + "Remove an object from a shard.", + "remove \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 5); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + on_subgroup_type(cmd_tokens[1], remove, capi, cmd_tokens[2] /*key*/, subgroup_index, shard_index); + return true; + }}, + {"op_remove", + "Remove an object from an object pool.", + "op_remove \n" + "Please note that cascade automatically decides the object pool path using the key's prefix.", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 2); + op_remove(capi, cmd_tokens[1]); + return true; + }}, + {"get", + "Get an object (by version).", + "get [ version(default:current version) ]\n" + "type := " SUBGROUP_TYPE_LIST "\n" + "stable := 0|1 using stable data or not.", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 6); + bool stable = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[5], nullptr, 0)); + persistent::version_t version = CURRENT_VERSION; + if(cmd_tokens.size() >= 7) { + version = static_cast(std::stol(cmd_tokens[6], nullptr, 0)); + } + on_subgroup_type(cmd_tokens[1], get, capi, cmd_tokens[2], version, stable, subgroup_index, shard_index); + return true; + }}, + {"op_get", + "Get an object from an object pool (by version).", + "op_get [ version(default:current version) ]\n" + "stable := 0|1 using stable data or not.\n" + "Please note that cascade automatically decides the object pool path using the key's prefix.", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 3); + bool stable = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + persistent::version_t version = CURRENT_VERSION; + if(cmd_tokens.size() >= 4) { + version = static_cast(std::stol(cmd_tokens[3], nullptr, 0)); + } + auto res = capi.get(cmd_tokens[1], version, stable); + check_get_result(res); + return true; + }}, + {"get_by_time", + "Get an object (by timestamp in microseconds).", + "get_by_time \n" + "type := " SUBGROUP_TYPE_LIST "\n" + "stable := 0|1 using stable data or not", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 7); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + uint64_t ts_us = static_cast(std::stol(cmd_tokens[5], nullptr, 0)); + bool stable = static_cast(std::stoi(cmd_tokens[6], nullptr, 0)); + on_subgroup_type(cmd_tokens[1], get_by_time, capi, cmd_tokens[2], ts_us, stable, subgroup_index, shard_index); + return true; + }}, + {"op_get_by_time", + "Get an object from an object pool (by timestamp in microseconds).", + "op_get_by_time \n" + "stable := 0|1 using stable data or not\n" + "Please note that cascade automatically decides the object pool path using the key's prefix.", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 4); + uint64_t ts_us = static_cast(std::stol(cmd_tokens[2], nullptr, 0)); + bool stable = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + auto res = capi.get_by_time(cmd_tokens[1], ts_us, stable); + check_get_result(res); + return true; + }}, + {"multi_get", + "Get an object, which will participate atomic broadcast for the latest value.", + "multi_get \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 5); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + on_subgroup_type(cmd_tokens[1], multi_get, capi, cmd_tokens[2], subgroup_index, shard_index); + return true; + }}, + {"op_multi_get", + "Get an object, which will participate atomic broadcast for the latest value.", + "op_multi_get \n", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 2); + auto res = capi.multi_get(cmd_tokens[1]); + check_get_result(res); + return true; + }}, + {"multi_get_size", + "Get the size of an object, which will participate atomic broadcast for the latest size.", + "multi_get_size \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 5); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + on_subgroup_type(cmd_tokens[1], multi_get_size, capi, cmd_tokens[2], subgroup_index, shard_index); + return true; + }}, + {"op_multi_get_size", + "Get the size of an object, which will participate atomic broadcast for the latest size.", + "op_multi_get_size \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 2); + auto res = capi.multi_get_size(cmd_tokens[1]); + check_get_result(res); + return true; + }}, + {"get_size", + "Get the size of an object (by version).", + "get_size [ version(default:current version) ]\n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 6); + bool stable = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[5], nullptr, 0)); + persistent::version_t version = CURRENT_VERSION; + if(cmd_tokens.size() >= 7) { + version = static_cast(std::stol(cmd_tokens[6], nullptr, 0)); + } + on_subgroup_type(cmd_tokens[1], get_size, capi, cmd_tokens[2], version, stable, subgroup_index, shard_index); + return true; + }}, + {"op_get_size", + "Get the size of an object from an object pool (by version).", + "op_get_size [ version(default:current version) ]\n" + "Please note that cascade automatically decides the object pool path using the key's prefix.", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 3); + persistent::version_t version = CURRENT_VERSION; + bool stable = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + if(cmd_tokens.size() >= 4) { + version = static_cast(std::stol(cmd_tokens[3], nullptr, 0)); + } + auto res = capi.get_size(cmd_tokens[1], version, stable); + check_get_result(res); + return true; + }}, + {"get_size_by_time", + "Get the size of an object (by timestamp in microseconds).", + "get_size_by_time \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 7); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + uint64_t ts_us = static_cast(std::stol(cmd_tokens[5], nullptr, 0)); + bool stable = static_cast(std::stoi(cmd_tokens[6], nullptr, 0)); + on_subgroup_type(cmd_tokens[1], get_size_by_time, capi, cmd_tokens[2], ts_us, stable, subgroup_index, shard_index); + return true; + }}, + {"op_get_size_by_time", + "Get the size of an object from an object pool (by timestamp in microseconds).", + "op_get_size_by_time \n" + "Please note that cascade automatically decides the object pool path using the key's prefix.", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 4); + uint64_t ts_us = static_cast(std::stol(cmd_tokens[2], nullptr, 0)); + bool stable = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + auto res = capi.get_size_by_time(cmd_tokens[1], ts_us, stable); + check_get_result(res); + return true; + }}, + {"multi_list_keys", + "list the object keys in a shard using atomic broadcast for the latest version.", + "multi_list_keys \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 4); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + on_subgroup_type(cmd_tokens[1], multi_list_keys, capi, subgroup_index, shard_index); + return true; + }}, + {"op_multi_list_keys", + "list the object keys in a shard using atomic broadcast for the latest version.", + "op_multi_list_keys \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 2); + auto result = capi.multi_list_keys(cmd_tokens[1]); + check_op_list_keys_result(capi.wait_list_keys(result)); + return true; + }}, + {"list_keys", + "list the object keys in a shard (by version).", + "list_keys [ version(default:current version) ]\n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 5); + bool stable = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + persistent::version_t version = CURRENT_VERSION; + if(cmd_tokens.size() >= 6) { + version = static_cast(std::stol(cmd_tokens[5], nullptr, 0)); + } + on_subgroup_type(cmd_tokens[1], list_keys, capi, version, stable, subgroup_index, shard_index); + return true; + }}, + {"op_list_keys", + "list the object keys in an object pool (by version).", + "op_list_keys [ version(default:current version) ]\n", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 3); + bool stable = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + persistent::version_t version = CURRENT_VERSION; + if(cmd_tokens.size() >= 4) { + version = static_cast(std::stol(cmd_tokens[3], nullptr, 0)); + } + auto result = capi.list_keys(version, stable, cmd_tokens[1]); + check_op_list_keys_result(capi.wait_list_keys(result)); + return true; + }}, + {"list_keys_by_time", + "list the object keys in a shard (by timestamp in mircoseconds).", + "list_keys_by_time \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 6); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + uint64_t ts_us = static_cast(std::stoull(cmd_tokens[4], nullptr, 0)); + bool stable = static_cast(std::stoi(cmd_tokens[5], nullptr, 0)); + on_subgroup_type(cmd_tokens[1], list_keys_by_time, capi, ts_us, stable, subgroup_index, shard_index); + return true; + }}, + {"op_list_keys_by_time", + "list the object keys in an object pool (by timestamp in microseconds).", + "op_list_keys_by_time \n", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 4); + uint64_t ts_us = static_cast(std::stoull(cmd_tokens[2], nullptr, 0)); + bool stable = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + auto result = capi.list_keys_by_time(ts_us, stable, cmd_tokens[1]); + check_op_list_keys_result(capi.wait_list_keys(result)); + return true; + }}, #ifdef HAS_BOOLINQ - { - "LINQ Tester Commands", "", "", command_handler_t() - }, - { - "list_data_by_prefix", - "LINQ API Tester: list the object with a specific prefix", - "list_data_by_prefix [ version(default:current version) ] \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,5); - const std::string& prefix = cmd_tokens[2]; - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - persistent::version_t version = CURRENT_VERSION; - if (cmd_tokens.size() >= 6) { - version = static_cast(std::stol(cmd_tokens[5],nullptr,0)); - } - on_subgroup_type(cmd_tokens[1],list_data_by_prefix,capi,prefix,version,subgroup_index,shard_index); - return true; - } - }, - { - "list_data_between_versions", - "LINQ API Tester: list an object data between versions", - "list_data_between_versions [ start version(default:MIN) ] [ end version (default:MAX) ] \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,5); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - - persistent::version_t version_start = INVALID_VERSION; - persistent::version_t version_end = INVALID_VERSION; - if (cmd_tokens.size() >= 6) { - version_start = static_cast(std::stol(cmd_tokens[5],nullptr,0)); - } - if (cmd_tokens.size() >= 7) { - version_end = static_cast(std::stol(cmd_tokens[6],nullptr,0)); - } - on_subgroup_type(cmd_tokens[1], list_data_between_versions, capi, cmd_tokens[2], subgroup_index, shard_index, version_start, version_end); - return true; - } - }, - { - "list_data_between_timestamps", - "LINQ API Tester: list an object data between points of time", - "list_data_between_timestamps [ start time(default:MIN) ] [ end time (default:MAX) ] \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,5); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3],nullptr,0)); - uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4],nullptr,0)); - - uint64_t start = 0; - uint64_t end = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); - if (cmd_tokens.size() >= 6) { - start = static_cast(std::stol(cmd_tokens[5],nullptr,0)); - } - if (cmd_tokens.size() >= 7) { - end = static_cast(std::stol(cmd_tokens[6],nullptr,0)); - } - on_subgroup_type(cmd_tokens[1], list_data_between_timestamps, capi, cmd_tokens[2], subgroup_index, shard_index, start, end); - return true; - } - }, - { - "list_data_in_subgroup", - "LINQ API Tester: list all objects in a subgroup", - "list_data_in_subgroup [ version (default:CURRENT_VERSION) ] \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,3); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - - persistent::version_t version = INVALID_VERSION; - if (cmd_tokens.size() >= 4) { - version = static_cast(std::stol(cmd_tokens[3],nullptr,0)); - } - on_subgroup_type(cmd_tokens[1], list_data_in_subgroup, capi, subgroup_index, version); - return true; - } - }, -#endif// HAS_BOOLINQ - { - "Notification Test Commands","","",command_handler_t() - }, - { - "op_register_notification", - "Register a notification to an object pool", - "op_reigster_notification ", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens, 2); - bool ret = capi.register_notification_handler( - [](const Blob& msg)->void{ - std::cout << "Object Pool Notification received:" - << "data:" << std::string(reinterpret_cast(msg.bytes),msg.size) - << std::endl; - }, - cmd_tokens[1]); - std::cout << "Notification Registered to object pool:" << cmd_tokens[1] - << ". Old handler replaced? " << ret << std::endl; - return true; - } - }, - { - "op_unregister_notification", - "Unregister a notification from an object pool", - "op_unreigster_notification ", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens, 2); - bool ret = capi.register_notification_handler({}, - cmd_tokens[1]); - std::cout << "Notification Unregistered from object pool:" << cmd_tokens[1] - << ". Old handler replaced? " << ret << std::endl; - return true; - } - }, - { - "register_notification", - "Register a notification handler to a subgroup", - "register_notification \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,3); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - - bool ret = false; - on_subgroup_type(cmd_tokens[1], ret = register_notification, capi, subgroup_index); - - std::cout << "Notification Registered to Subgroup " << cmd_tokens[1] << ":" << subgroup_index - << ". Old handler replaced?" << ret << std::endl; - return true; - } - }, - { - "unregister_notification", - "Unregister a notification handler from a subgroup", - "unregister_notification \n" - "type := " SUBGROUP_TYPE_LIST, - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,3); - uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2],nullptr,0)); - - bool ret = false; - on_subgroup_type(cmd_tokens[1], ret = unregister_notification, capi, subgroup_index); - - std::cout << "Notification Registered to Subgroup " << cmd_tokens[1] << ":" << subgroup_index - << ". Old handler replaced?" << ret << std::endl; - return true; - } - }, + {"LINQ Tester Commands", "", "", command_handler_t()}, + {"list_data_by_prefix", + "LINQ API Tester: list the object with a specific prefix", + "list_data_by_prefix [ version(default:current version) ] \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 5); + const std::string& prefix = cmd_tokens[2]; + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + persistent::version_t version = CURRENT_VERSION; + if(cmd_tokens.size() >= 6) { + version = static_cast(std::stol(cmd_tokens[5], nullptr, 0)); + } + on_subgroup_type(cmd_tokens[1], list_data_by_prefix, capi, prefix, version, subgroup_index, shard_index); + return true; + }}, + {"list_data_between_versions", + "LINQ API Tester: list an object data between versions", + "list_data_between_versions [ start version(default:MIN) ] [ end version (default:MAX) ] \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 5); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + + persistent::version_t version_start = INVALID_VERSION; + persistent::version_t version_end = INVALID_VERSION; + if(cmd_tokens.size() >= 6) { + version_start = static_cast(std::stol(cmd_tokens[5], nullptr, 0)); + } + if(cmd_tokens.size() >= 7) { + version_end = static_cast(std::stol(cmd_tokens[6], nullptr, 0)); + } + on_subgroup_type(cmd_tokens[1], list_data_between_versions, capi, cmd_tokens[2], subgroup_index, shard_index, version_start, version_end); + return true; + }}, + {"list_data_between_timestamps", + "LINQ API Tester: list an object data between points of time", + "list_data_between_timestamps [ start time(default:MIN) ] [ end time (default:MAX) ] \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 5); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[3], nullptr, 0)); + uint32_t shard_index = static_cast(std::stoi(cmd_tokens[4], nullptr, 0)); + + uint64_t start = 0; + uint64_t end = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + if(cmd_tokens.size() >= 6) { + start = static_cast(std::stol(cmd_tokens[5], nullptr, 0)); + } + if(cmd_tokens.size() >= 7) { + end = static_cast(std::stol(cmd_tokens[6], nullptr, 0)); + } + on_subgroup_type(cmd_tokens[1], list_data_between_timestamps, capi, cmd_tokens[2], subgroup_index, shard_index, start, end); + return true; + }}, + {"list_data_in_subgroup", + "LINQ API Tester: list all objects in a subgroup", + "list_data_in_subgroup [ version (default:CURRENT_VERSION) ] \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 3); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + + persistent::version_t version = INVALID_VERSION; + if(cmd_tokens.size() >= 4) { + version = static_cast(std::stol(cmd_tokens[3], nullptr, 0)); + } + on_subgroup_type(cmd_tokens[1], list_data_in_subgroup, capi, subgroup_index, version); + return true; + }}, +#endif // HAS_BOOLINQ + {"Notification Test Commands", "", "", command_handler_t()}, + {"op_register_notification", + "Register a notification to an object pool", + "op_reigster_notification ", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 2); + bool ret = capi.register_notification_handler( + [](const Blob& msg) -> void { + std::cout << "Object Pool Notification received:" + << "data:" << std::string(reinterpret_cast(msg.bytes), msg.size) + << std::endl; + }, + cmd_tokens[1]); + std::cout << "Notification Registered to object pool:" << cmd_tokens[1] + << ". Old handler replaced? " << ret << std::endl; + return true; + }}, + {"op_unregister_notification", + "Unregister a notification from an object pool", + "op_unreigster_notification ", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 2); + bool ret = capi.register_notification_handler({}, + cmd_tokens[1]); + std::cout << "Notification Unregistered from object pool:" << cmd_tokens[1] + << ". Old handler replaced? " << ret << std::endl; + return true; + }}, + {"register_notification", + "Register a notification handler to a subgroup", + "register_notification \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 3); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + + bool ret = false; + on_subgroup_type(cmd_tokens[1], ret = register_notification, capi, subgroup_index); + + std::cout << "Notification Registered to Subgroup " << cmd_tokens[1] << ":" << subgroup_index + << ". Old handler replaced?" << ret << std::endl; + return true; + }}, + {"unregister_notification", + "Unregister a notification handler from a subgroup", + "unregister_notification \n" + "type := " SUBGROUP_TYPE_LIST, + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 3); + uint32_t subgroup_index = static_cast(std::stoi(cmd_tokens[2], nullptr, 0)); + + bool ret = false; + on_subgroup_type(cmd_tokens[1], ret = unregister_notification, capi, subgroup_index); + + std::cout << "Notification Registered to Subgroup " << cmd_tokens[1] << ":" << subgroup_index + << ". Old handler replaced?" << ret << std::endl; + return true; + }}, #ifdef ENABLE_EVALUATION - { - "Performance Test Commands","","",command_handler_t() - }, - { - "perftest_object_pool", - "Performance Tester for put to an object pool.", - "perftest_object_pool [, ...] \n" - "type := " SUBGROUP_TYPE_LIST "\n" - "put_type := put|put_and_forget|trigger_put \n" - "'member selection policy' refers how the external clients pick a member in a shard;\n" - " Available options: FIXED|RANDOM|ROUNDROBIN;\n" - "'r/w ratio' is the ratio of get vs put operations, INF for all put test; \n" - "'max rate' is the maximum number of operations in Operations per Second, 0 for best effort; \n" - "'duration' is the span of the whole experiments; \n" - "'clientn' is a host[:port] pair representing the parallel clients. The port is default to " + std::to_string(PERFTEST_PORT), - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,9); - - PutType put_type = PutType::PUT; - - if (cmd_tokens[2] == "put_and_forget") { - put_type = PutType::PUT_AND_FORGET; - } else if (cmd_tokens[2] == "trigger_put") { - put_type = PutType::TRIGGER_PUT; - } - - std::string object_pool_pathname = cmd_tokens[3]; - ExternalClientToCascadeServerMapping member_selection_policy = FIXED; - if (cmd_tokens[4] == "RANDOM") { - member_selection_policy = ExternalClientToCascadeServerMapping::RANDOM; - } else if (cmd_tokens[4] == "ROUNDROBIN") { - member_selection_policy = ExternalClientToCascadeServerMapping::ROUNDROBIN; - } - double read_write_ratio = std::stod(cmd_tokens[5]); - uint64_t max_rate = std::stoul(cmd_tokens[6],nullptr,0); - uint64_t duration_sec = std::stoul(cmd_tokens[7],nullptr,0); - - PerfTestClient ptc{capi}; - uint32_t pos = 8; - while (pos < cmd_tokens.size()) { - std::string::size_type colon_pos = cmd_tokens[pos].find(':'); - if (colon_pos == std::string::npos) { - ptc.add_or_update_server(cmd_tokens[pos],PERFTEST_PORT); - } else { - ptc.add_or_update_server(cmd_tokens[pos].substr(0,colon_pos), - static_cast(std::stoul(cmd_tokens[pos].substr(colon_pos+1),nullptr,0))); - } - pos ++; - } - bool ret = false; - on_subgroup_type(cmd_tokens[1], ret = perftest, ptc, put_type, object_pool_pathname, member_selection_policy,read_write_ratio,max_rate,duration_sec,"timestamp.log"); - return ret; - } - }, - { - "perftest_shard", - "Performance Tester for put to a shard.", - "perftest_shard [, ...] \n" - "type := " SUBGROUP_TYPE_LIST "\n" - "put_type := put|put_and_forget|trigger_put \n" - "'member selection policy' refers how the external clients pick a member in a shard;\n" - " Available options: FIXED|RANDOM|ROUNDROBIN;\n" - "'r/w ratio' is the ratio of get vs put operations, INF for all put test; \n" - "'max rate' is the maximum number of operations in Operations per Second, 0 for best effort; \n" - "'duration' is the span of the whole experiments; \n" - "'clientn' is a host[:port] pair representing the parallel clients. The port is default to " + std::to_string(PERFTEST_PORT), - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,10); - - PutType put_type = PutType::PUT; - - if (cmd_tokens[2] == "put_and_forget") { - put_type = PutType::PUT_AND_FORGET; - } else if (cmd_tokens[2] == "trigger_put") { - put_type = PutType::TRIGGER_PUT; - } - - uint32_t subgroup_index = std::stoul(cmd_tokens[3],nullptr,0); - uint32_t shard_index = std::stoul(cmd_tokens[4],nullptr,0); - - ExternalClientToCascadeServerMapping member_selection_policy = FIXED; - if (cmd_tokens[5] == "RANDOM") { - member_selection_policy = ExternalClientToCascadeServerMapping::RANDOM; - } else if (cmd_tokens[5] == "ROUNDROBIN") { - member_selection_policy = ExternalClientToCascadeServerMapping::ROUNDROBIN; - } - double read_write_ratio = std::stod(cmd_tokens[6]); - uint64_t max_rate = std::stoul(cmd_tokens[7],nullptr,0); - uint64_t duration_sec = std::stoul(cmd_tokens[8],nullptr,0); - - PerfTestClient ptc{capi}; - uint32_t pos = 9; - while (pos < cmd_tokens.size()) { - std::string::size_type colon_pos = cmd_tokens[pos].find(':'); - if (colon_pos == std::string::npos) { - ptc.add_or_update_server(cmd_tokens[pos],PERFTEST_PORT); - } else { - ptc.add_or_update_server(cmd_tokens[pos].substr(0,colon_pos), - static_cast(std::stoul(cmd_tokens[pos].substr(colon_pos+1),nullptr,0))); - } - pos ++; - } - bool ret = false; - on_subgroup_type(cmd_tokens[1], ret = perftest,ptc,put_type,subgroup_index,shard_index,member_selection_policy,read_write_ratio,max_rate,duration_sec,"output.log"); - return ret; - } - }, - { - "perftest_ordered_put", - "Performance Test for ordered_put in a shard.", - "perftest_ordered_put \n" - "type := " SUBGROUP_TYPE_LIST "\n" - "'duration_sec' is the span of the whole experiments", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,6); - uint32_t message_size = std::stoul(cmd_tokens[2],nullptr,0); - uint64_t duration_sec = std::stoul(cmd_tokens[3],nullptr,0); - uint32_t subgroup_index = std::stoul(cmd_tokens[4],nullptr,0); - uint32_t shard_index = std::stoul(cmd_tokens[5],nullptr,0); - - on_subgroup_type(cmd_tokens[1], perftest_ordered_put, capi, message_size, duration_sec, subgroup_index, shard_index); - return true; - } - }, - { - "dump_timestamp", - "Dump timestamp for a given shard. Each node will write its timestamps to the given file.", - "dump_timestamp \n" - "type := " SUBGROUP_TYPE_LIST "\n" - "filename := timestamp log filename", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,5); - uint32_t subgroup_index = std::stoul(cmd_tokens[2],nullptr,0); - uint32_t shard_index = std::stoul(cmd_tokens[3],nullptr,0); - on_subgroup_type(cmd_tokens[1]/*subgroup type*/, dump_timestamp, capi, subgroup_index, shard_index, cmd_tokens[4]/*filename*/); - return true; - } - }, - { - "op_dump_timestamp", - "Dump timestamps for a given object pool. Each node will write its timestamps to the given file.", - "op_dump_timestamp \n" - "filename := timestamp log filename", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,3); - capi.dump_timestamp(cmd_tokens[2],cmd_tokens[1]); - TimestampLogger::flush(cmd_tokens[2]); - return true; - } - }, + {"Performance Test Commands", "", "", command_handler_t()}, + {"perftest_object_pool", + "Performance Tester for put to an object pool.", + "perftest_object_pool [, ...] \n" + "type := " SUBGROUP_TYPE_LIST "\n" + "put_type := put|put_and_forget|trigger_put \n" + "'member selection policy' refers how the external clients pick a member in a shard;\n" + " Available options: FIXED|RANDOM|ROUNDROBIN;\n" + "'r/w ratio' is the ratio of get vs put operations, INF for all put test; \n" + "'max rate' is the maximum number of operations in Operations per Second, 0 for best effort; \n" + "'duration' is the span of the whole experiments; \n" + "'clientn' is a host[:port] pair representing the parallel clients. The port is default to " + + std::to_string(PERFTEST_PORT), + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 9); + + PutType put_type = PutType::PUT; + + if(cmd_tokens[2] == "put_and_forget") { + put_type = PutType::PUT_AND_FORGET; + } else if(cmd_tokens[2] == "trigger_put") { + put_type = PutType::TRIGGER_PUT; + } + + std::string object_pool_pathname = cmd_tokens[3]; + ExternalClientToCascadeServerMapping member_selection_policy = FIXED; + if(cmd_tokens[4] == "RANDOM") { + member_selection_policy = ExternalClientToCascadeServerMapping::RANDOM; + } else if(cmd_tokens[4] == "ROUNDROBIN") { + member_selection_policy = ExternalClientToCascadeServerMapping::ROUNDROBIN; + } + double read_write_ratio = std::stod(cmd_tokens[5]); + uint64_t max_rate = std::stoul(cmd_tokens[6], nullptr, 0); + uint64_t duration_sec = std::stoul(cmd_tokens[7], nullptr, 0); + + PerfTestClient ptc{capi}; + uint32_t pos = 8; + while(pos < cmd_tokens.size()) { + std::string::size_type colon_pos = cmd_tokens[pos].find(':'); + if(colon_pos == std::string::npos) { + ptc.add_or_update_server(cmd_tokens[pos], PERFTEST_PORT); + } else { + ptc.add_or_update_server(cmd_tokens[pos].substr(0, colon_pos), + static_cast(std::stoul(cmd_tokens[pos].substr(colon_pos + 1), nullptr, 0))); + } + pos++; + } + bool ret = false; + on_subgroup_type(cmd_tokens[1], ret = perftest, ptc, put_type, object_pool_pathname, member_selection_policy, read_write_ratio, max_rate, duration_sec, "timestamp.log"); + return ret; + }}, + {"perftest_shard", + "Performance Tester for put to a shard.", + "perftest_shard [, ...] \n" + "type := " SUBGROUP_TYPE_LIST "\n" + "put_type := put|put_and_forget|trigger_put \n" + "'member selection policy' refers how the external clients pick a member in a shard;\n" + " Available options: FIXED|RANDOM|ROUNDROBIN;\n" + "'r/w ratio' is the ratio of get vs put operations, INF for all put test; \n" + "'max rate' is the maximum number of operations in Operations per Second, 0 for best effort; \n" + "'duration' is the span of the whole experiments; \n" + "'clientn' is a host[:port] pair representing the parallel clients. The port is default to " + + std::to_string(PERFTEST_PORT), + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 10); + + PutType put_type = PutType::PUT; + + if(cmd_tokens[2] == "put_and_forget") { + put_type = PutType::PUT_AND_FORGET; + } else if(cmd_tokens[2] == "trigger_put") { + put_type = PutType::TRIGGER_PUT; + } + + uint32_t subgroup_index = std::stoul(cmd_tokens[3], nullptr, 0); + uint32_t shard_index = std::stoul(cmd_tokens[4], nullptr, 0); + + ExternalClientToCascadeServerMapping member_selection_policy = FIXED; + if(cmd_tokens[5] == "RANDOM") { + member_selection_policy = ExternalClientToCascadeServerMapping::RANDOM; + } else if(cmd_tokens[5] == "ROUNDROBIN") { + member_selection_policy = ExternalClientToCascadeServerMapping::ROUNDROBIN; + } + double read_write_ratio = std::stod(cmd_tokens[6]); + uint64_t max_rate = std::stoul(cmd_tokens[7], nullptr, 0); + uint64_t duration_sec = std::stoul(cmd_tokens[8], nullptr, 0); + + PerfTestClient ptc{capi}; + uint32_t pos = 9; + while(pos < cmd_tokens.size()) { + std::string::size_type colon_pos = cmd_tokens[pos].find(':'); + if(colon_pos == std::string::npos) { + ptc.add_or_update_server(cmd_tokens[pos], PERFTEST_PORT); + } else { + ptc.add_or_update_server(cmd_tokens[pos].substr(0, colon_pos), + static_cast(std::stoul(cmd_tokens[pos].substr(colon_pos + 1), nullptr, 0))); + } + pos++; + } + bool ret = false; + on_subgroup_type(cmd_tokens[1], ret = perftest, ptc, put_type, subgroup_index, shard_index, member_selection_policy, read_write_ratio, max_rate, duration_sec, "output.log"); + return ret; + }}, + {"perftest_ordered_put", + "Performance Test for ordered_put in a shard.", + "perftest_ordered_put \n" + "type := " SUBGROUP_TYPE_LIST "\n" + "'duration_sec' is the span of the whole experiments", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 6); + uint32_t message_size = std::stoul(cmd_tokens[2], nullptr, 0); + uint64_t duration_sec = std::stoul(cmd_tokens[3], nullptr, 0); + uint32_t subgroup_index = std::stoul(cmd_tokens[4], nullptr, 0); + uint32_t shard_index = std::stoul(cmd_tokens[5], nullptr, 0); + + on_subgroup_type(cmd_tokens[1], perftest_ordered_put, capi, message_size, duration_sec, subgroup_index, shard_index); + return true; + }}, + {"dump_timestamp", + "Dump timestamp for a given shard. Each node will write its timestamps to the given file.", + "dump_timestamp \n" + "type := " SUBGROUP_TYPE_LIST "\n" + "filename := timestamp log filename", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 5); + uint32_t subgroup_index = std::stoul(cmd_tokens[2], nullptr, 0); + uint32_t shard_index = std::stoul(cmd_tokens[3], nullptr, 0); + on_subgroup_type(cmd_tokens[1] /*subgroup type*/, dump_timestamp, capi, subgroup_index, shard_index, cmd_tokens[4] /*filename*/); + return true; + }}, + {"op_dump_timestamp", + "Dump timestamps for a given object pool. Each node will write its timestamps to the given file.", + "op_dump_timestamp \n" + "filename := timestamp log filename", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 3); + capi.dump_timestamp(cmd_tokens[2], cmd_tokens[1]); + TimestampLogger::flush(cmd_tokens[2]); + return true; + }}, #endif }; inline void do_command(ServiceClientAPI& capi, const std::vector& cmd_tokens) { try { ssize_t command_index = find_command(commands, cmd_tokens[0]); - if (command_index>=0) { - if (commands.at(command_index).handler(capi,cmd_tokens)) { + if(command_index >= 0) { + if(commands.at(command_index).handler(capi, cmd_tokens)) { std::cout << "-> Succeeded." << std::endl; } else { std::cout << "-> Failed." << std::endl; @@ -1704,50 +1580,50 @@ inline void do_command(ServiceClientAPI& capi, const std::vector& c } else { print_red("unknown command:" + cmd_tokens[0]); } - } catch (const derecho::derecho_exception &ex) { - print_red (std::string("Exception:") + ex.what()); - } catch (...) { - print_red ("Unknown exception caught."); + } catch(const derecho::derecho_exception& ex) { + print_red(std::string("Exception:") + ex.what()); + } catch(...) { + print_red("Unknown exception caught."); } } void interactive_test(ServiceClientAPI& capi) { // loop - while (shell_is_active) { + while(shell_is_active) { char* malloced_cmd = readline("cmd> "); std::string cmdline(malloced_cmd); free(malloced_cmd); - if (cmdline == "")continue; + if(cmdline == "") continue; add_history(cmdline.c_str()); std::string delimiter = " "; - do_command(capi,tokenize(cmdline, delimiter.c_str())); + do_command(capi, tokenize(cmdline, delimiter.c_str())); } std::cout << "Client exits." << std::endl; } void detached_test(ServiceClientAPI& capi, int argc, char** argv) { std::vector cmd_tokens; - for(int i=1;i///// * "mount-point" is where the cascade data is mounted. @@ -32,13 +34,22 @@ using namespace derecho::cascade; using FuseClientContextType = FuseClientContext; -#define FCC(p) static_cast(p) -#define FCC_REQ(req) FCC(fuse_req_userdata(req)) +// #define FCC(p) static_cast(p) +// #define fcc(req) FCC(fuse_req_userdata(req)) + +static FuseClientContextType* fcc(fuse_req_t req) { + return static_cast(fuse_req_userdata(req)); +} static void fs_init(void* userdata, struct fuse_conn_info* conn) { dbg_default_trace("entering {}.", __func__); + auto fcc = static_cast(userdata); + // TODO ? where is truncating handled in open syscall? + int can_trunc = conn->capable & FUSE_CAP_ATOMIC_O_TRUNC; + dbg_default_info("can trunc: {}", can_trunc); + if(derecho::hasCustomizedConfKey(CONF_LAYOUT_JSON_LAYOUT)) { - FCC(userdata)->initialize(json::parse(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT))); + fcc->initialize(json::parse(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT))); } else if(derecho::hasCustomizedConfKey(CONF_LAYOUT_JSON_LAYOUT_FILE)) { nlohmann::json layout_array; std::ifstream json_file(derecho::getAbsoluteFilePath(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT_FILE))); @@ -47,7 +58,7 @@ static void fs_init(void* userdata, struct fuse_conn_info* conn) { throw derecho::derecho_exception("Cannot load json configuration from file."); } json_file >> layout_array; - FCC(userdata)->initialize(layout_array); + fcc->initialize(layout_array); } dbg_default_trace("leaving {}.", __func__); } @@ -57,23 +68,47 @@ static void fs_destroy(void* userdata) { dbg_default_trace("leaving {}.", __func__); } +static int do_lookup(fuse_req_t req, fuse_ino_t parent, const char* name, + struct fuse_entry_param* e) { + // TODO: make this more efficient by implement a dedicated call in FCC. + auto name_to_ino = fcc(req)->get_dir_entries(parent); + if(name_to_ino.find(name) == name_to_ino.end()) { + return ENOENT; + } + // TODO: change timeout settings. + e->ino = name_to_ino.at(name); + e->attr_timeout = 10000.0; + e->entry_timeout = 10000.0; + e->attr.st_ino = e->ino; + fcc(req)->fill_stbuf_by_ino(e->attr); + + return 0; +} + static void fs_lookup(fuse_req_t req, fuse_ino_t parent, const char* name) { dbg_default_trace("entering {}.", __func__); struct fuse_entry_param e; - // TODO: make this more efficient by implement a dedicated call in FCC. - auto name_to_ino = FCC_REQ(req)->get_dir_entries(parent); - if(name_to_ino.find(name) == name_to_ino.end()) { - fuse_reply_err(req, ENOENT); + + int err = do_lookup(req, parent, name, &e); + if(err) { + fuse_reply_err(req, err); } else { - // TODO: change timeout settings. - e.ino = name_to_ino.at(name); - e.attr_timeout = 10000.0; - e.entry_timeout = 10000.0; - e.attr.st_ino = e.ino; - FCC_REQ(req)->fill_stbuf_by_ino(e.attr); fuse_reply_entry(req, &e); } + // auto name_to_ino = fcc(req)->get_dir_entries(parent); + // if(name_to_ino.find(name) == name_to_ino.end()) { + // fuse_reply_err(req, ENOENT); + // } else { + // // TODO: change timeout settings. + // e.ino = name_to_ino.at(name); + // e.attr_timeout = 10000.0; + // e.entry_timeout = 10000.0; + // e.attr.st_ino = e.ino; + // fcc(req)->fill_stbuf_by_ino(e.attr); + // fuse_reply_entry(req, &e); + // } + dbg_default_trace("leaving {}.", __func__); } @@ -84,7 +119,7 @@ static void fs_getattr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi std::memset(&stbuf, 0, sizeof(stbuf)); stbuf.st_ino = ino; - FCC_REQ(req)->fill_stbuf_by_ino(stbuf); + fcc(req)->fill_stbuf_by_ino(stbuf); fuse_reply_attr(req, &stbuf, 10000.0); dbg_default_trace("leaving {}.", __func__); @@ -103,7 +138,7 @@ static void dirbuf_add(fuse_req_t req, struct dirbuf* b, const char* name, fuse_ b->p = (char*)realloc(b->p, b->size); std::memset(&stbuf, 0, sizeof(stbuf)); stbuf.st_ino = ino; - FCC_REQ(req)->fill_stbuf_by_ino(stbuf); // additional effect + fcc(req)->fill_stbuf_by_ino(stbuf); // additional effect fuse_add_direntry(req, b->p + oldsize, b->size - oldsize, name, &stbuf, b->size); } @@ -112,7 +147,7 @@ static int reply_buf_limited(fuse_req_t req, const char* buf, size_t bufsize, if(static_cast(off) < bufsize) return fuse_reply_buf(req, buf + off, std::min(bufsize - off, maxsize)); else - return fuse_reply_buf(req, NULL, 0); + return fuse_reply_buf(req, nullptr, 0); } static void fs_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, struct fuse_file_info* fi) { @@ -121,7 +156,7 @@ static void fs_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, s std::memset(&b, 0, sizeof(b)); dirbuf_add(req, &b, ".", 1); dirbuf_add(req, &b, "..", 1); - for(auto kv : FCC_REQ(req)->get_dir_entries(ino)) { + for(auto kv : fcc(req)->get_dir_entries(ino)) { dirbuf_add(req, &b, kv.first.c_str(), kv.second); } reply_buf_limited(req, b.p, b.size, off, size); @@ -132,10 +167,10 @@ static void fs_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, s static void fs_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { dbg_default_trace("entering {}.", __func__); int err; - if((fi->flags & O_ACCMODE) != O_RDONLY && !FCC_REQ(req)->is_writable(ino)) { + if((fi->flags & O_ACCMODE) != O_RDONLY && !fcc(req)->is_writable(ino)) { // allow write for key nodes fuse_reply_err(req, EACCES); - } else if((err = FCC_REQ(req)->open_file(ino, fi)) != 0) { + } else if((err = fcc(req)->open_file(ino, fi)) != 0) { fuse_reply_err(req, err); } else { dbg_default_debug("fi({:x})->fh={:x}", reinterpret_cast(fi), fi->fh); @@ -147,15 +182,22 @@ static void fs_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { static void fs_read(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, struct fuse_file_info* fi) { dbg_default_trace("entering {}.", __func__); - FileBytes* pfb = reinterpret_cast(fi->fh); - reply_buf_limited(req, reinterpret_cast(pfb->bytes), pfb->size, off, size); + // TODO `file_bytes.bytes` might be freed in reply buf once fs_read ends. may need to go back to storing in `fi` + FileBytes file_bytes; + int res = fcc(req)->read_file(ino, size, off, fi, &file_bytes); + if(res == 0) { + reply_buf_limited(req, reinterpret_cast(file_bytes.bytes.data()), file_bytes.bytes.size(), off, size); + } else { + // TODO handle error + fuse_reply_err(req, 1); + } dbg_default_trace("leaving {}.", __func__); } static void fs_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { dbg_default_trace("entering {}.", __func__); - FCC_REQ(req)->close_file(ino, fi); + fcc(req)->close_file(ino, fi); fuse_reply_err(req, 0); dbg_default_trace("leaving {}.", __func__); } @@ -165,7 +207,7 @@ static void fs_write(fuse_req_t req, fuse_ino_t ino, const char* buf, size_t siz dbg_default_trace("entering {}.", __func__); // TODO write_buf instead?... use fi?? - ssize_t res = FCC_REQ(req)->write_file(ino, buf, size, off, fi); + ssize_t res = fcc(req)->write_file(ino, buf, size, off, fi); if(res < 0) { fuse_reply_err(req, -res); } else { @@ -175,28 +217,99 @@ static void fs_write(fuse_req_t req, fuse_ino_t ino, const char* buf, size_t siz dbg_default_trace("leaving {}.", __func__); } +static void fs_setattr(fuse_req_t req, fuse_ino_t ino, struct stat* attr, + int valid, struct fuse_file_info* fi) { + dbg_default_trace("setattr(ino={%u}, valid: {%o}).", ino, valid); + + // char procname[64]; + // struct lo_inode* inode = lo_inode(req, ino); + // int ifd = inode->fd; + int res; + + try { + if(!fi) { + errno = EACCES; + throw 1; + } + if(valid & FUSE_SET_ATTR_SIZE) { + dbg_default_trace("attr_size(length={})", (unsigned long)attr->st_size); + + if(!fcc(req)->is_writable(ino)) { + errno = EACCES; + throw 1; + } + res = fcc(req)->truncate(ino, attr->st_size, fi); + + if(res == -1) { + errno = EACCES; + throw 1; + } + } + // if(valid & FUSE_SET_ATTR_MODE) {} + // if(valid & (FUSE_SET_ATTR_UID | FUSE_SET_ATTR_GID)) {} + // if(valid & (FUSE_SET_ATTR_ATIME | FUSE_SET_ATTR_MTIME)) {} + return fs_getattr(req, ino, fi); + } catch(int& ex) { + int saverr = errno; + fuse_reply_err(req, saverr); + } +} + +static void fs_create(fuse_req_t req, fuse_ino_t parent, const char* name, + mode_t mode, struct fuse_file_info* fi) { + struct fuse_entry_param e; + int err; + + dbg_default_info("fs create: {}", name); + // TODO fi fields like direct_io and keep_cache + + err = fcc(req)->open_at(parent, name, mode, fi); + if(err) { + return (void)fuse_reply_err(req, err); + } + + err = do_lookup(req, parent, name, &e); + if(err) + fuse_reply_err(req, err); + else + fuse_reply_create(req, &e, fi); +} + +static void fs_mknod(fuse_req_t req, fuse_ino_t parent, const char* name, + mode_t mode, dev_t rdev) { + dbg_default_info("fs mknod: {}", name); + fuse_reply_err(req, 1); +} + +static void fs_mkdir(fuse_req_t req, fuse_ino_t parent, const char* name, + mode_t mode) { + dbg_default_info("fs mkdir: {}", name); + fuse_reply_err(req, 1); +} + static const struct fuse_lowlevel_ops fs_ops = { .init = fs_init, .destroy = fs_destroy, .lookup = fs_lookup, - .forget = NULL, + .forget = nullptr, .getattr = fs_getattr, - .setattr = NULL, - .readlink = NULL, - .mknod = NULL, - .mkdir = NULL, - .unlink = NULL, - .rmdir = NULL, - .symlink = NULL, - .rename = NULL, - .link = NULL, + .setattr = fs_setattr, + .readlink = nullptr, + .mknod = fs_mknod, + .mkdir = fs_mkdir, + .unlink = nullptr, + .rmdir = nullptr, + .symlink = nullptr, + .rename = nullptr, + .link = nullptr, .open = fs_open, .read = fs_read, .write = fs_write, .release = fs_release, - .fsync = NULL, - .opendir = NULL, + .fsync = nullptr, + .opendir = nullptr, .readdir = fs_readdir, + .create = fs_create, }; /** @@ -230,6 +343,7 @@ void prepare_derecho_conf_file() { int main(int argc, char** argv) { prepare_derecho_conf_file(); + // TODO extra args like cache level / specifying timeout struct fuse_args args = FUSE_ARGS_INIT(argc, argv); struct fuse_session* se = nullptr; struct fuse_cmdline_opts opts; diff --git a/src/service/fuse/fuse_client_context.hpp b/src/service/fuse/fuse_client_context.hpp index f2b4b4c0..e2c4ac1a 100644 --- a/src/service/fuse/fuse_client_context.hpp +++ b/src/service/fuse/fuse_client_context.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -47,20 +48,27 @@ typedef enum { class FileBytes { public: - size_t size; - uint8_t* bytes; - FileBytes() : size(0), bytes(nullptr) {} - FileBytes(size_t s) : size(s) { - bytes = nullptr; - if(s > 0) { - bytes = (uint8_t*)malloc(s); - } - } - virtual ~FileBytes() { - if(bytes) { - free(bytes); - } - } + // TODO use Blob instead??? + std::vector bytes; + + // static FileBytes* from_fh(uint64_t fh) { + // return reinterpret_cast(fh); + // } + // public: + // size_t size; + // uint8_t* bytes; + // FileBytes() : size(0), bytes(nullptr) {} + // FileBytes(size_t s) : size(s) { + // bytes = nullptr; + // if(s > 0) { + // bytes = (uint8_t*)malloc(s); + // } + // } + // virtual ~FileBytes() { + // if(bytes) { + // free(bytes); + // } + // } }; class FuseClientINode { @@ -89,6 +97,10 @@ class FuseClientINode { return sizeof(*this); } + virtual uint64_t open_at(const char* name, mode_t mode, struct fuse_file_info* fi) { + return 0; + } + virtual uint64_t read_file(FileBytes* fb) { (void)fb; return 0; @@ -103,6 +115,10 @@ class FuseClientINode { return -1; } + virtual uint64_t truncate(size_t size) { + return -1; + } + virtual void initialize() { } @@ -119,9 +135,46 @@ class FuseClientINode { } } - virtual void write_helper(const char* buf, size_t size, off_t off, - struct fuse_file_info* fi, FileBytes* file_bytes) { - bool appending = false; +private: + // Helper functions for check_update() + virtual void update_contents() { + return; + } +}; + +class FuseDataINode : public FuseClientINode { +public: + FileBytes cached_data; + persistent::version_t version; + uint64_t timestamp_us; + persistent::version_t previous_version; + // previous version by key, INVALID_VERSION for the first value of the key. + persistent::version_t previous_version_by_key; + + ServiceClientAPI& capi; + + FuseDataINode(ServiceClientAPI& capi) : cached_data(FileBytes()), capi(capi) { + } + + virtual uint64_t get_file_size() override { + check_update(); + return cached_data.bytes.size(); + } + + virtual uint64_t read_file(FileBytes* file_bytes) override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + + check_update(); + file_bytes->bytes = cached_data.bytes; + + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return 0; + } + + virtual uint64_t write_file(const char* buf, size_t size, off_t off, + struct fuse_file_info* fi) { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + update_contents(); if(fi) { std::stringstream ss; @@ -134,34 +187,50 @@ class FuseClientINode { } if(fi->flags & O_APPEND) { dbg_default_info("appending"); - appending = true; - } else { - dbg_default_info("not appending, resizing TODO ?? "); + // TODO faster write ?? } - if(fi->flags & O_TRUNC) { + if(fi->flags & O_TRUNC) { // TODO handled in open not here dbg_default_info("truncating"); } } - size_t next_size = std::max(file_bytes->size, off + size); - - if(!appending) { - next_size = off + size; // remove extra old content - } + size_t next_size = std::max(cached_data.bytes.size(), off + size); + // if(!appending) { // TODO truncating happens elsewhere + // next_size = off + size; // remove extra old content + // } dbg_default_info("writing. off: {}, buf_size: {}, next_size: {}.", off, size, next_size); dbg_default_info("writing content: {}.", std::quoted(reinterpret_cast(buf))); - FileBytes next_bytes(next_size); - memcpy(next_bytes.bytes, file_bytes->bytes, next_size); - memcpy(next_bytes.bytes + off, buf, size); - dbg_default_info("next file: {}.", std::quoted(reinterpret_cast(next_bytes.bytes))); - write_contents(next_bytes.bytes, next_size); + FileBytes contents = cached_data; + contents.bytes.resize(next_size); + memcpy(contents.bytes.data() + off, buf, size); + // FileBytes next_bytes(next_size); + // memcpy(next_bytes.bytes, file_bytes->bytes, next_size); + // memcpy(next_bytes.bytes + off, buf, size); - return; + dbg_default_info("next file: {}.", std::quoted(reinterpret_cast(contents.bytes.data()))); + write_contents(contents.bytes.data(), next_size); + + update_contents(); // TODO race conditions ... ?? + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return 0; + } + + virtual uint64_t truncate(size_t size) { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + update_contents(); + + dbg_default_info("truncating to {}.", size); + FileBytes contents = cached_data; + contents.bytes.resize(size); + write_contents(contents.bytes.data(), size); + + update_contents(); // TODO race conditions ... ?? + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return 0; } private: - // Helper functions for check_update() virtual void update_contents() { return; } @@ -250,7 +319,7 @@ class RootMetaINode : public FuseClientINode { std::string contents; public: - RootMetaINode(ServiceClientAPI& _capi) : capi(_capi) { + RootMetaINode(ServiceClientAPI& capi) : capi(capi) { this->update_interval = 2; this->last_update_sec = 0; this->type = INodeType::META; @@ -283,8 +352,9 @@ class RootMetaINode : public FuseClientINode { virtual uint64_t read_file(FileBytes* file_bytes) override { this->check_update(); - file_bytes->size = contents.size(); - file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); + file_bytes->bytes.assign(contents.begin(), contents.end()); + // file_bytes->size = contents.size(); + // file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); return 0; } }; @@ -308,7 +378,7 @@ class ShardINode : public FuseClientINode { ServiceClientAPI& capi; std::map key_to_ino; - ShardINode(uint32_t shidx, fuse_ino_t pino, ServiceClientAPI& _capi) : shard_index(shidx), capi(_capi) { + ShardINode(uint32_t shidx, fuse_ino_t pino, ServiceClientAPI& capi) : shard_index(shidx), capi(capi) { this->type = INodeType::SHARD; this->display_name = "shard-" + std::to_string(shidx); this->parent = pino; @@ -349,7 +419,7 @@ class ShardINode : public FuseClientINode { ServiceClientAPI& capi; std::map key_to_ino; - ShardINode(uint32_t shidx, fuse_ino_t pino, ServiceClientAPI& _capi) : shard_index(shidx), capi(_capi) { + ShardINode(uint32_t shidx, fuse_ino_t pino, ServiceClientAPI& capi) : shard_index(shidx), capi(capi) { this->type = INodeType::SHARD; this->display_name = "shard-" + std::to_string(shidx); this->parent = pino; @@ -417,8 +487,8 @@ class ShardMetaINode : public FuseClientINode { } public: - ShardMetaINode(const uint32_t _shard_index, const uint32_t _subgroup_index, ServiceClientAPI& _capi) - : shard_index(_shard_index), subgroup_index(_subgroup_index), capi(_capi) { + ShardMetaINode(const uint32_t shard_index, const uint32_t subgroup_index, ServiceClientAPI& capi) + : shard_index(shard_index), subgroup_index(subgroup_index), capi(capi) { this->update_interval = 2; this->last_update_sec = 0; this->type = INodeType::META; @@ -432,26 +502,20 @@ class ShardMetaINode : public FuseClientINode { virtual uint64_t read_file(FileBytes* file_bytes) override { check_update(); - file_bytes->size = contents.size(); - file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); + file_bytes->bytes.assign(contents.begin(), contents.end()); + // file_bytes->size = contents.size(); + // file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); return 0; } }; template -class KeyINode : public FuseClientINode { +class KeyINode : public FuseDataINode { public: typename CascadeType::KeyType key; - std::unique_ptr file_bytes; - uint64_t file_size; - persistent::version_t version; - uint64_t timestamp_us; - persistent::version_t previous_version; - persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. - ServiceClientAPI& capi; - KeyINode(typename CascadeType::KeyType& k, fuse_ino_t pino, ServiceClientAPI& _capi) - : key(k), file_bytes(std::make_unique()), capi(_capi) { + KeyINode(typename CascadeType::KeyType& k, fuse_ino_t pino, ServiceClientAPI& capi) + : FuseDataINode(capi), key(k) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); this->update_interval = 2; this->last_update_sec = 0; @@ -470,33 +534,6 @@ class KeyINode : public FuseClientINode { dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } - virtual uint64_t read_file(FileBytes* _file_bytes) override { - dbg_default_trace("[{}]entering {}.", gettid(), __func__); - check_update(); - _file_bytes->size = this->file_bytes.get()->size; - _file_bytes->bytes = static_cast(malloc(this->file_bytes.get()->size)); - memcpy(_file_bytes->bytes, this->file_bytes.get()->bytes, this->file_bytes.get()->size); - dbg_default_trace("[{}]leaving {}.", gettid(), __func__); - return 0; - } - - virtual uint64_t write_file(const char* buf, size_t size, off_t off, struct fuse_file_info* fi) override { - dbg_default_trace("[{}]entering {}.", gettid(), __func__); - - check_update(); - - write_helper(buf, size, off, fi, this->file_bytes.get()); - - dbg_default_trace("[{}]leaving {}.", gettid(), __func__); - - return 0; - } - - virtual uint64_t get_file_size() override { - check_update(); - return this->file_bytes.get()->size; - } - KeyINode(KeyINode&& fci) { this->type = std::move(fci.type); this->display_name = std::move(fci.display_name); @@ -505,12 +542,6 @@ class KeyINode : public FuseClientINode { this->capi = std::move(fci.capi); } - virtual ~KeyINode() { - dbg_default_info("[{}] entering {}.", gettid(), __func__); - file_bytes.reset(nullptr); - dbg_default_info("[{}] leaving {}.", gettid(), __func__); - } - private: virtual void update_contents() override { ShardINode* pino_shard = reinterpret_cast*>(this->parent); @@ -536,14 +567,19 @@ class KeyINode : public FuseClientINode { this->previous_version_by_key = access_ptr->previous_version_by_key; blob = access_ptr->blob; } else { - this->file_bytes.get()->size = mutils::bytes_size(reply); - this->file_bytes.get()->bytes = static_cast(malloc(this->file_bytes.get()->size)); - reply.to_bytes(this->file_bytes.get()->bytes); + // this->file_bytes.get()->size = mutils::bytes_size(reply); + // this->file_bytes.get()->bytes = static_cast(malloc(this->file_bytes.get()->size)); + // reply.to_bytes(this->file_bytes.get()->bytes); + this->cached_data.bytes.resize(mutils::bytes_size(reply)); + reply.to_bytes(this->cached_data.bytes.data()); return; } - file_bytes.get()->size = blob.size; - file_bytes.get()->bytes = static_cast(malloc(file_bytes.get()->size)); - memcpy(file_bytes.get()->bytes, blob.bytes, file_bytes.get()->size); + // file_bytes.get()->size = blob.size; + // file_bytes.get()->bytes = static_cast(malloc(file_bytes.get()->size)); + // memcpy(file_bytes.get()->bytes, blob.bytes, file_bytes.get()->size); + this->cached_data.bytes.resize(blob.size); + memcpy(this->cached_data.bytes.data(), blob.bytes, blob.size); + return; return; } } @@ -557,7 +593,7 @@ class KeyINode : public FuseClientINode { // TODO obj.previous_version = INVALID_VERSION; // this->previous_version; obj.previous_version_by_key = INVALID_VERSION; // this->previous_version_by_key; - obj.blob = Blob(content, size); + obj.blob = Blob(content, size, true); // TODO safe to emplace? dbg_default_debug("raw put"); derecho::rpc::QueryResults> result @@ -568,8 +604,6 @@ class KeyINode : public FuseClientINode { dbg_default_debug("node({}) replied with version:{},ts_us:{}", reply_future.first, std::get<0>(reply), std::get<1>(reply)); } - - update_contents(); // TODO race conditions ... ?? } }; @@ -646,8 +680,8 @@ class ObjectPoolMetaINode : public FuseClientINode { } public: - ObjectPoolMetaINode(const std::string _cur_pathname, ServiceClientAPI& _capi) - : cur_pathname(_cur_pathname), capi(_capi) { + ObjectPoolMetaINode(const std::string cur_pathname, ServiceClientAPI& capi) + : cur_pathname(cur_pathname), capi(capi) { this->update_interval = 2; this->last_update_sec = 0; this->type = INodeType::META; @@ -670,8 +704,9 @@ class ObjectPoolMetaINode : public FuseClientINode { virtual uint64_t read_file(FileBytes* file_bytes) override { check_update(); - file_bytes->size = contents.size(); - file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); + file_bytes->bytes.assign(contents.begin(), contents.end()); + // file_bytes->size = contents.size(); + // file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); return 0; } }; @@ -684,7 +719,7 @@ class ObjectPoolPathINode : public FuseClientINode { std::set key_children; std::set objp_children; - ObjectPoolPathINode(fuse_ino_t pino, ServiceClientAPI& _capi) : capi(_capi) { + ObjectPoolPathINode(fuse_ino_t pino, ServiceClientAPI& capi) : capi(capi) { this->update_interval = 10; this->last_update_sec = 0; this->type = INodeType::OBJECTPOOL_PATH; @@ -694,8 +729,8 @@ class ObjectPoolPathINode : public FuseClientINode { this->children.emplace_back(std::make_unique(cur_pathname, capi)); } - ObjectPoolPathINode(std::string _cur_pathname, fuse_ino_t pino, ServiceClientAPI& _capi) - : capi(_capi), cur_pathname(_cur_pathname) { + ObjectPoolPathINode(std::string cur_pathname, fuse_ino_t pino, ServiceClientAPI& capi) + : capi(capi), cur_pathname(cur_pathname) { this->update_interval = 10; this->last_update_sec = 10; this->type = INodeType::OBJECTPOOL_PATH; @@ -830,6 +865,7 @@ class ObjectPoolPathINode : public FuseClientINode { ++it; } } + return; } virtual std::map get_dir_entries() override { @@ -839,6 +875,32 @@ class ObjectPoolPathINode : public FuseClientINode { return FuseClientINode::get_dir_entries(); } + virtual uint64_t open_at(const char* name, mode_t mode, struct fuse_file_info* fi) { + dbg_default_info("object pool open at"); + update_contents(); + if(!this->is_object_pool) { + return 1; + } + + ObjectWithStringKey obj; + obj.key = cur_pathname + "/" + name; // TODO replace with std path + obj.previous_version = CURRENT_VERSION; + obj.previous_version_by_key = CURRENT_VERSION; + obj.blob = Blob(); + + dbg_default_debug("object pool put"); + derecho::rpc::QueryResults> result = capi.put(obj); + + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + dbg_default_debug("node({}) replied with version:{},ts_us:{}", + reply_future.first, std::get<0>(reply), std::get<1>(reply)); + } + + update_contents(); + return 0; + } + private: virtual void update_contents() override { update_objpINodes(); @@ -848,7 +910,7 @@ class ObjectPoolPathINode : public FuseClientINode { class ObjectPoolRootINode : public ObjectPoolPathINode { public: - ObjectPoolRootINode(ServiceClientAPI& _capi, fuse_ino_t pino = FUSE_ROOT_ID) : ObjectPoolPathINode(pino, _capi) { + ObjectPoolRootINode(ServiceClientAPI& capi, fuse_ino_t pino = FUSE_ROOT_ID) : ObjectPoolPathINode(pino, capi) { this->display_name = "ObjectPools"; this->parent = pino; } @@ -870,18 +932,12 @@ class ObjectPoolRootINode : public ObjectPoolPathINode { } }; -class ObjectPoolKeyINode : public FuseClientINode { +class ObjectPoolKeyINode : public FuseDataINode { public: std::string key; - std::unique_ptr file_bytes; - persistent::version_t version; - uint64_t timestamp_us; - persistent::version_t previous_version; - persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. - ServiceClientAPI& capi; - ObjectPoolKeyINode(std::string k, fuse_ino_t pino, ServiceClientAPI& _capi) - : key(k), file_bytes(std::make_unique()), capi(_capi) { + ObjectPoolKeyINode(std::string k, fuse_ino_t pino, ServiceClientAPI& capi) + : FuseDataINode(capi), key(k) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); this->update_interval = 2; this->last_update_sec = 0; @@ -892,40 +948,6 @@ class ObjectPoolKeyINode : public FuseClientINode { dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } - virtual uint64_t read_file(FileBytes* _file_bytes) override { - dbg_default_debug("-- READ FILE of key:[{}], [{}]entering {}.", this->key, gettid(), __func__); - check_update(); - _file_bytes->size = this->file_bytes.get()->size; - _file_bytes->bytes = static_cast(malloc(this->file_bytes.get()->size)); - memcpy(_file_bytes->bytes, this->file_bytes.get()->bytes, this->file_bytes.get()->size); - dbg_default_debug("[{}]leaving {}.", gettid(), __func__); - return 0; - } - - virtual uint64_t write_file(const char* buf, size_t size, off_t off, struct fuse_file_info* fi) override { - dbg_default_trace("[{}]entering {}.", gettid(), __func__); - - check_update(); - - write_helper(buf, size, off, fi, this->file_bytes.get()); - - dbg_default_trace("[{}]leaving {}.", gettid(), __func__); - - return 0; - } - - virtual uint64_t get_file_size() override { - dbg_default_debug("----GET FILE SIZE key is [{}].", this->key); - check_update(); - return this->file_bytes.get()->size; - } - - virtual ~ObjectPoolKeyINode() { - dbg_default_info("[{}] entering {}.", gettid(), __func__); - file_bytes.reset(nullptr); - dbg_default_info("[{}] leaving {}.", gettid(), __func__); - } - private: virtual void update_contents() override { dbg_default_debug("\n \n ----OBJP keyInode key is:[{}] - update content [{}] entering {}.", this->key, gettid(), __func__); @@ -939,9 +961,11 @@ class ObjectPoolKeyINode : public FuseClientINode { this->previous_version = access_ptr->previous_version; this->previous_version_by_key = access_ptr->previous_version_by_key; blob = access_ptr->blob; - this->file_bytes.get()->size = blob.size; - this->file_bytes.get()->bytes = static_cast(malloc(blob.size)); - memcpy(this->file_bytes.get()->bytes, blob.bytes, blob.size); + // this->file_bytes.get()->size = blob.size; + // this->file_bytes.get()->bytes = static_cast(malloc(blob.size)); + // memcpy(this->file_bytes.get()->bytes, blob.bytes, blob.size); + this->cached_data.bytes.resize(blob.size); + memcpy(this->cached_data.bytes.data(), blob.bytes, blob.size); return; } dbg_default_trace("\n \n ----OBJP keyInode update content [{}] leaving {}.", gettid(), __func__); @@ -952,7 +976,7 @@ class ObjectPoolKeyINode : public FuseClientINode { obj.key = this->key; obj.previous_version = INVALID_VERSION; obj.previous_version_by_key = INVALID_VERSION; - obj.blob = Blob(content, size); + obj.blob = Blob(content, size, true); // TODO safe to emplace? dbg_default_debug("object pool put"); derecho::rpc::QueryResults> result = capi.put(obj); @@ -962,8 +986,6 @@ class ObjectPoolKeyINode : public FuseClientINode { dbg_default_debug("node({}) replied with version:{},ts_us:{}", reply_future.first, std::get<0>(reply), std::get<1>(reply)); } - - update_contents(); // TODO race conditions ... ?? } }; @@ -971,7 +993,7 @@ class MetadataServiceRootINode : public FuseClientINode { ServiceClientAPI& capi; public: - MetadataServiceRootINode(ServiceClientAPI& _capi) : capi(_capi) { + MetadataServiceRootINode(ServiceClientAPI& capi) : capi(capi) { this->type = INodeType::METADATA_SERVICE; this->display_name = "MetadataService"; this->parent = FUSE_ROOT_ID; @@ -993,7 +1015,7 @@ class DataPathLogicRootINode : public FuseClientINode { ServiceClientAPI& capi; public: - DataPathLogicRootINode(ServiceClientAPI& _capi, fuse_ino_t pino) : capi(_capi) { + DataPathLogicRootINode(ServiceClientAPI& capi, fuse_ino_t pino) : capi(capi) { this->type = INodeType::DATAPATH_LOGIC; this->display_name = "DataPathLogic"; this->parent = FUSE_ROOT_ID; @@ -1014,11 +1036,11 @@ class DLLINode : public FuseClientINode { public: std::string file_name; // DLL id? ServiceClientAPI& capi; - DLLINode(std::string& _filename, fuse_ino_t pino, ServiceClientAPI& _capi) - : file_name(_filename), capi(_capi) { + DLLINode(std::string& file_name, fuse_ino_t pino, ServiceClientAPI& capi) + : file_name(file_name), capi(capi) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); this->type = INodeType::DLL; - this->display_name = std::string("dllfile") + _filename; + this->display_name = std::string("dllfile") + file_name; this->parent = pino; dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } @@ -1148,10 +1170,10 @@ class FuseClientContext { double timeout_sec = 1.0; // TO_FOREVER; // 1 - common attributes stbuf.st_dev = FUSE_CLIENT_DEV_ID; - stbuf.st_nlink = 1; + stbuf.st_nlink = 1; // TODO set property stbuf.st_uid = getuid(); stbuf.st_gid = getgid(); - stbuf.st_atim = this->init_timestamp; + stbuf.st_atim = this->init_timestamp; // TODO set properly stbuf.st_mtim = this->init_timestamp; stbuf.st_ctim = this->init_timestamp; // 2 - special attributes @@ -1234,12 +1256,27 @@ class FuseClientContext { return pfci->type == INodeType::KEY; } + int open_at(fuse_ino_t parent, const char* name, mode_t mode, struct fuse_file_info* fi) { + FuseClientINode* pfci = reinterpret_cast(parent); + + return pfci->open_at(name, mode, fi); + } + + int read_file(fuse_ino_t ino, size_t size, off_t off, struct fuse_file_info* fi, FileBytes* file_bytes) { + FuseClientINode* pfci = reinterpret_cast(ino); + + size_t result = pfci->read_file(file_bytes); // TODO only read necessary? + + return result; // wrong number container size + } + int write_file(fuse_ino_t ino, const char* buf, size_t size, off_t off, struct fuse_file_info* fi) { dbg_default_trace("[{}]entering {} with ino={:x}.", gettid(), __func__, ino); if(!is_writable(ino)) { return -EACCES; } + FuseClientINode* pfci = reinterpret_cast(ino); pfci->write_file(buf, size, off, fi); // TODO errors?? @@ -1249,6 +1286,17 @@ class FuseClientContext { return result; } + int truncate(fuse_ino_t ino, size_t size, struct fuse_file_info* fi) { + // TODO use errno ? + FuseClientINode* pfci = reinterpret_cast(ino); + + if(!is_writable(ino)) { + return -1; + } + + return pfci->truncate(size); + } + /** * open a file. * @param ino inode @@ -1258,12 +1306,27 @@ class FuseClientContext { int open_file(fuse_ino_t ino, struct fuse_file_info* fi) { dbg_default_trace("[{}]entering {} with ino={:x}.", gettid(), __func__, ino); FuseClientINode* pfci = reinterpret_cast(ino); + + if(fi->flags & O_CREAT) { + dbg_default_info("open: create flag"); + } + // TODO handle truncate + + if(fi->flags & O_TRUNC) { + dbg_default_info("open: trunc flag"); + pfci->truncate(0); + } + if(fi->flags & O_LARGEFILE) { + dbg_default_info("open: large file flag"); // TODO !! + } + if(pfci->type != INodeType::KEY && pfci->type != INodeType::META) { return EISDIR; } - FileBytes* fb = new FileBytes(); - pfci->read_file(fb); - fi->fh = reinterpret_cast(fb); + // TODO cleaner to not set fh to something non file handle like ?? + // FileBytes* fb = new FileBytes(); + // pfci->read_file(fb); + // fi->fh = reinterpret_cast(fb); dbg_default_trace("[{}]leaving {}.", gettid(), __func__); return 0; } @@ -1276,10 +1339,11 @@ class FuseClientContext { */ int close_file(fuse_ino_t ino, struct fuse_file_info* fi) { dbg_default_trace("[{}]entering {} with ino={:x}.", gettid(), __func__, ino); - void* pfb = reinterpret_cast(fi->fh); - if(pfb != nullptr) { - delete static_cast(pfb); - } + // TODO no longer need to delete? + // void* pfb = reinterpret_cast(fi->fh); + // if(pfb != nullptr) { + // delete static_cast(pfb); + // } return 0; dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } From 39a4bec8b3d18068c495d2380a7ca2fe065c3abc Mon Sep 17 00:00:00 2001 From: Michael Ye Date: Sat, 28 Jan 2023 15:41:15 -0500 Subject: [PATCH 06/10] feat: mostly complete high-level fuse api --- src/service/client.cpp | 25 +- src/service/fuse/CMakeLists.txt | 10 + src/service/fuse/fcc_hl.hpp | 310 +++++++++++++++ src/service/fuse/fuse_client.cpp | 6 +- src/service/fuse/fuse_client_context.hpp | 11 +- src/service/fuse/fuse_client_hl.cpp | 479 +++++++++++++++++++++++ src/service/fuse/path_tree.hpp | 158 ++++++++ 7 files changed, 986 insertions(+), 13 deletions(-) create mode 100644 src/service/fuse/fcc_hl.hpp create mode 100644 src/service/fuse/fuse_client_hl.cpp create mode 100644 src/service/fuse/path_tree.hpp diff --git a/src/service/client.cpp b/src/service/client.cpp index e437e701..17e4baf8 100644 --- a/src/service/client.cpp +++ b/src/service/client.cpp @@ -740,14 +740,18 @@ ssize_t find_command(const std::vector& command_list, const std return pos; } +void print_help(const std::string& cmd); + bool shell_is_active = true; #define SUBGROUP_TYPE_LIST "VCSS|PCSS|TCSS" #define SHARD_MEMBER_SELECTION_POLICY_LIST "FirstMember|LastMember|Random|FixedRandom|RoundRobin|KeyHashing|UserSpecified" -#define CHECK_FORMAT(tks, argc) \ - if(tks.size() < argc) { \ - print_red("Invalid command format. Please try help " + tks[0] + "."); \ - return false; \ +#define CHECK_FORMAT(tks, argc) \ + if(tks.size() < argc) { \ + print_red("Invalid command format."); \ + print_help(tks[0]); \ + return false; \ } + std::vector commands = { {"General Commands", "", "", command_handler_t()}, {"help", @@ -1568,6 +1572,15 @@ std::vector commands = { #endif }; +void print_help(const std::string& cmd) { + ssize_t command_index = find_command(commands, cmd); + if(command_index < 0) { + print_red("unknown command:'" + cmd + "'."); + } else { + print_red(commands.at(command_index).help); + } +} + inline void do_command(ServiceClientAPI& capi, const std::vector& cmd_tokens) { try { ssize_t command_index = find_command(commands, cmd_tokens[0]); @@ -1590,7 +1603,9 @@ inline void do_command(ServiceClientAPI& capi, const std::vector& c void interactive_test(ServiceClientAPI& capi) { // loop while(shell_is_active) { - char* malloced_cmd = readline("cmd> "); + char* malloced_cmd = readline("\033[1;36m" + "cmd> " + "\033[0m"); std::string cmdline(malloced_cmd); free(malloced_cmd); if(cmdline == "") continue; diff --git a/src/service/fuse/CMakeLists.txt b/src/service/fuse/CMakeLists.txt index 6770ff1d..78de76f6 100644 --- a/src/service/fuse/CMakeLists.txt +++ b/src/service/fuse/CMakeLists.txt @@ -13,6 +13,16 @@ if (${HAS_FUSE}) $ ) + add_executable(fuse_client_hl fuse_client_hl.cpp) + target_include_directories(fuse_client_hl PRIVATE + $ + $ + $ + $ + ) + target_link_libraries(fuse_client_hl cascade readline fuse3 fuse_client_signals) + set_target_properties(fuse_client_hl PROPERTIES OUTPUT_NAME cascade_fuse_client_hl) + add_executable(fuse_client fuse_client.cpp) target_include_directories(fuse_client PRIVATE $ diff --git a/src/service/fuse/fcc_hl.hpp b/src/service/fuse/fcc_hl.hpp new file mode 100644 index 00000000..de05312f --- /dev/null +++ b/src/service/fuse/fcc_hl.hpp @@ -0,0 +1,310 @@ +#pragma once + +#define FUSE_USE_VERSION 34 + +#include +#include +#include +#include +#include +#include + +#include "path_tree.hpp" + +namespace fs = std::filesystem; + +struct timespec GLOBAL_INIT_TIMESTAMP; + +std::shared_ptr DL; + +enum NodeFlag { + OP_PREFIX_DIR = 1 << 0, + OP_ROOT_DIR = 1 << 1, + OP_KEY_DIR = 1 << 2, + OP_KEY = 1 << 3, + OP_INFO = 1 << 4, +}; + +const int OP_DIR = OP_PREFIX_DIR | OP_ROOT_DIR | OP_KEY_DIR; + +struct NodeData { + NodeFlag flag; + // bool is_dir; + // bool is_op_prefix; + struct timespec init_timestamp; + std::vector bytes; + + NodeData(const NodeFlag flag) : flag(flag) { + init_timestamp = GLOBAL_INIT_TIMESTAMP; + } + + NodeData(const NodeFlag flag, const std::string& contents) : flag(flag) { + init_timestamp = GLOBAL_INIT_TIMESTAMP; + bytes = std::vector(contents.begin(), contents.end()); + } +}; + +struct FSTree { + using Node = PathTree; + + Node* root; + + FSTree() { + root = new Node("/", NodeData(OP_PREFIX_DIR)); + } + + ~FSTree() { delete root; } + + Node* add_op_root(const fs::path& path) { + // TODO add info + return root->set(path, NodeData(OP_PREFIX_DIR), NodeData(OP_ROOT_DIR)); + } + + Node* add_op_key(const fs::path& path) { + // invariant: assumes op_root already exists + return root->set(path, NodeData(OP_KEY_DIR), NodeData(OP_KEY)); + } + + Node* add_op_key_dir(const fs::path& path) { + // invariant: assumes op_root already exists + return root->set(path, NodeData(OP_KEY_DIR), NodeData(OP_KEY_DIR)); + } + + void new_op_paths(const std::vector paths) { + auto old_root = root; + root = new Node("/", NodeData(OP_PREFIX_DIR)); + for(const auto& path : paths) { + auto op_root = add_op_root(path); + if(op_root == nullptr) { + continue; + // TODO errors + } + auto old_op_root = old_root->extract(path); + if(old_op_root != nullptr) { + Node::replace(op_root, old_op_root); + } + } + delete old_root; + } + + static Node* object_pool_root(Node* node) { + if(node == nullptr || node->data.flag & OP_PREFIX_DIR) { + return nullptr; + } + if(node->parent == nullptr || node->data.flag & OP_ROOT_DIR) { + return node; + } + return object_pool_root(node->parent); + } + + Node* object_pool_root(const fs::path& path) { + auto node = root->get_while_valid(path); + return object_pool_root(node); + } + + static int get_stat(Node* node, struct stat* stbuf) { + if(node == nullptr) { + return -ENOENT; + } + // not needed: st_dev, st_blksize, st_ino (unless use_ino mount option) + // TODO maintain stbuf in data? + stbuf->st_nlink = 1; + stbuf->st_uid = fuse_get_context()->uid; + stbuf->st_gid = fuse_get_context()->gid; + stbuf->st_atim = node->data.init_timestamp; // TODO set properly + stbuf->st_mtim = node->data.init_timestamp; + stbuf->st_ctim = node->data.init_timestamp; + // - at prefix dir location add .info file ??? + + if(node->data.flag & OP_DIR) { + if(node->data.flag & OP_PREFIX_DIR) { + stbuf->st_mode = S_IFDIR | 0555; + } else { + stbuf->st_mode = S_IFDIR | 0755; + } + stbuf->st_nlink = 2; // TODO calculate properly + for(const auto& [_, v] : node->children) { + stbuf->st_nlink += v->data.flag & OP_DIR; + } + } else { + // TODO somehow even when 0444, can still write ??? + stbuf->st_mode = S_IFREG | 0744; + stbuf->st_size = node->data.bytes.size(); + } + return 0; + } +}; + +namespace derecho { +namespace cascade { + +struct FuseClientContext { + ServiceClientAPI& capi; + + FSTree* tree; + + std::set local_dirs; + + time_t update_interval; + time_t last_update_sec; + + FuseClientContext() : capi(ServiceClientAPI::get_service_client()) { + DL = LoggerFactory::createLogger("fuse_client", spdlog::level::from_str(derecho::getConfString(CONF_LOGGER_DEFAULT_LOG_LEVEL))); + DL->set_pattern("[%T][%n][%^%l%$] %v"); + + clock_gettime(CLOCK_REALTIME, &GLOBAL_INIT_TIMESTAMP); + + update_interval = 15; + last_update_sec = 0; + + tree = new FSTree(); + + update_object_pools(); + } + + ~FuseClientContext() { + try { + dbg_debug(DL, "deleting tree"); + delete tree; + } catch(...) { + dbg_error(DL, "could not delete tree"); + } + } + + bool should_update() { + if(time(0) > last_update_sec + update_interval) { + return true; + } + return false; + } + + int put(const FSTree::Node* node) { + // invariant: node is file + ObjectWithStringKey obj; + obj.key = node->absolute_path(); + obj.previous_version = INVALID_VERSION; + obj.previous_version_by_key = INVALID_VERSION; + obj.blob = Blob(node->data.bytes.data(), node->data.bytes.size(), true); + // TODO verify emplaced avoids blob deleting data :( + + auto result = capi.put(obj); + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + dbg_info(DL, "node({}) replied with version:{},ts_us:{}", + reply_future.first, std::get<0>(reply), std::get<1>(reply)); + } + // TODO check for error + + return 0; + } + + void update_object_pools() { + auto reply = capi.list_object_pools(true); + std::vector paths(reply.size()); + for(size_t i = 0; i < reply.size(); ++i) { + paths[i] = fs::path(reply[i]); + } + + // TODO avoid re generating op keys? ... by not resetting tree + FSTree* old_tree = tree; + delete old_tree; + tree = new FSTree(); + + tree->new_op_paths(paths); + + for(const auto& op_root : paths) { + if(tree->root->get(op_root) != nullptr) { + auto keys = get_keys(op_root); + std::sort(keys.begin(), keys.end(), std::greater<>()); + for(const auto& k : keys) { + // TODO verify remove file colliding with directory here + auto key = tree->add_op_key(k); + if(key == nullptr) { + dbg_info(DL, "did not add {}", k); + } else { + key->data.bytes = get_contents(k); + // dbg_info(DL, "file: {}", std::quoted(reinterpret_cast(key->data.bytes.data()))); + } + } + } + } + + for(auto it = local_dirs.begin(); it != local_dirs.end();) { + if(tree->object_pool_root(*it) == nullptr) { + // TODO verify for op_root deleted + it = local_dirs.erase(it); + } else { + tree->add_op_key_dir(*it); + ++it; + } + } + + dbg_info(DL, "updating contents\n{}", string()); + + last_update_sec = time(0); + } + + std::string string() { + std::stringstream ss; + tree->root->print(100, ss); + return ss.str(); + } + + std::vector get_contents(const std::string& path) { + persistent::version_t version = CURRENT_VERSION; + auto result = capi.get(path, version, true); + + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + Blob blob = reply.blob; + std::vector bytes(blob.bytes, blob.bytes + blob.size); + // TODO!!! don't use .data() + // memcpy(bytes.data(), blob.bytes, blob.size); + return bytes; + } + return {}; + } + + std::vector get_keys(const std::string& path) { + persistent::version_t version = CURRENT_VERSION; + auto future_result = capi.list_keys(version, true, path); + auto reply = capi.wait_list_keys(future_result); + // TODO remove keys that collide with directory (key is a prefix of another key) + return reply; + } + + FSTree::Node* get(const std::string& path) { + if(should_update()) { + update_object_pools(); + // split between update object pool list and update keys + } + return tree->root->get(path); + } + // TODO use object pool root meta file to edit version # and such? + + // TODO cascade metaservice api. need: + // - get children given path. shouldnt be hard considering op_list_keys works from subdir of op root + // - get file metadata WITHOUT getting file contents (file size, modification time) + + /* + stat + + */ + // dev_t st_dev; /* ID of device containing file */ + // ino_t st_ino; /* Inode number */ + // mode_t st_mode; /* File type and mode */ + // nlink_t st_nlink; /* Number of hard links */ + // uid_t st_uid; /* User ID of owner */ + // gid_t st_gid; /* Group ID of owner */ + // dev_t st_rdev; /* Device ID (if special file) */ + // off_t st_size; /* Total size, in bytes */ + // blksize_t st_blksize; /* Block size for filesystem I/O */ + // blkcnt_t st_blocks; /* Number of 512B blocks allocated */ + + // struct timespec st_atim; /* Time of last access */ + // struct timespec st_mtim; /* Time of last modification */ + // struct timespec st_ctim; /* Time of last status change */ +}; + +} // namespace cascade +} // namespace derecho \ No newline at end of file diff --git a/src/service/fuse/fuse_client.cpp b/src/service/fuse/fuse_client.cpp index a6dd8354..a5316647 100644 --- a/src/service/fuse/fuse_client.cpp +++ b/src/service/fuse/fuse_client.cpp @@ -44,9 +44,7 @@ static FuseClientContextType* fcc(fuse_req_t req) { static void fs_init(void* userdata, struct fuse_conn_info* conn) { dbg_default_trace("entering {}.", __func__); auto fcc = static_cast(userdata); - // TODO ? where is truncating handled in open syscall? - int can_trunc = conn->capable & FUSE_CAP_ATOMIC_O_TRUNC; - dbg_default_info("can trunc: {}", can_trunc); + // TODO look through conn->capable options if(derecho::hasCustomizedConfKey(CONF_LAYOUT_JSON_LAYOUT)) { fcc->initialize(json::parse(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT))); @@ -114,7 +112,7 @@ static void fs_lookup(fuse_req_t req, fuse_ino_t parent, const char* name) { static void fs_getattr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { dbg_default_trace("entering {}.", __func__); - struct stat stbuf; + struct stat stbuf; // TODO memory leak? (void)fi; std::memset(&stbuf, 0, sizeof(stbuf)); diff --git a/src/service/fuse/fuse_client_context.hpp b/src/service/fuse/fuse_client_context.hpp index e2c4ac1a..29498d60 100644 --- a/src/service/fuse/fuse_client_context.hpp +++ b/src/service/fuse/fuse_client_context.hpp @@ -126,6 +126,7 @@ class FuseClientINode { void check_update() { struct timespec now; clock_gettime(CLOCK_REALTIME, &now); + // TODO why double if ??? if(now.tv_sec > (last_update_sec + update_interval)) { clock_gettime(CLOCK_REALTIME, &now); if(now.tv_sec > (last_update_sec + update_interval)) { @@ -144,7 +145,7 @@ class FuseClientINode { class FuseDataINode : public FuseClientINode { public: - FileBytes cached_data; + FileBytes cached_data; // TODO not necessary / bad idea since very expensive?? persistent::version_t version; uint64_t timestamp_us; persistent::version_t previous_version; @@ -165,6 +166,7 @@ class FuseDataINode : public FuseClientINode { dbg_default_trace("[{}]entering {}.", gettid(), __func__); check_update(); + // TODO necessary copy?? file_bytes->bytes = cached_data.bytes; dbg_default_trace("[{}]leaving {}.", gettid(), __func__); @@ -174,7 +176,7 @@ class FuseDataINode : public FuseClientINode { virtual uint64_t write_file(const char* buf, size_t size, off_t off, struct fuse_file_info* fi) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); - update_contents(); + check_update(); if(fi) { std::stringstream ss; @@ -218,7 +220,7 @@ class FuseDataINode : public FuseClientINode { virtual uint64_t truncate(size_t size) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); - update_contents(); + check_update(); dbg_default_info("truncating to {}.", size); FileBytes contents = cached_data; @@ -1309,12 +1311,13 @@ class FuseClientContext { if(fi->flags & O_CREAT) { dbg_default_info("open: create flag"); + // TODO get parent inode and create } // TODO handle truncate if(fi->flags & O_TRUNC) { dbg_default_info("open: trunc flag"); - pfci->truncate(0); + pfci->truncate(0); // TODO cache (do finally on close file) } if(fi->flags & O_LARGEFILE) { dbg_default_info("open: large file flag"); // TODO !! diff --git a/src/service/fuse/fuse_client_hl.cpp b/src/service/fuse/fuse_client_hl.cpp new file mode 100644 index 00000000..3b64c6c8 --- /dev/null +++ b/src/service/fuse/fuse_client_hl.cpp @@ -0,0 +1,479 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "fcc_hl.hpp" +#include "fuse_client_signals.hpp" +#include + +using namespace derecho::cascade; + +#define HAVE_SETXATTR + +struct cli_options { + const char* client_dir; + int update_interval; + int flag; +}; + +static cli_options options = { + .client_dir = nullptr, + .update_interval = 1, + .flag = 0, +}; + +#define OPTION(t, p) \ + { t, offsetof(struct cli_options, p), 1 } + +static const struct fuse_opt option_spec[] = { + OPTION("--client=%s", client_dir), + OPTION("--update-interval=%d", update_interval), + OPTION("--flag", flag), + FUSE_OPT_END}; + +static void show_help(const char* progname) { + printf("usage: %s [options] \n\n", progname); + printf(" --update-interval= Update-rate of file system contents\n" + " --client= Client directory\n" + " --flag Flag that does nothing\n" + "\n"); +} + +// --- + +static FuseClientContext* fcc() { + return static_cast(fuse_get_context()->private_data); +} + +static void* cascade_fs_init(struct fuse_conn_info* conn, + struct fuse_config* cfg) { + // TODO why read conf_layout_json_layout? + // TODO don't like no control over derecho config + return new FuseClientContext(); +} + +static void cascade_fs_destroy(void* private_data) { + delete static_cast(private_data); +} + +static int cascade_fs_getattr(const char* path, struct stat* stbuf, + struct fuse_file_info* fi) { + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + memset(stbuf, 0, sizeof(struct stat)); + return FSTree::get_stat(node, stbuf); +} + +// TODO :( invalid pointer dumped?? somehow cascade replys needs to be stored in a variable before +// result.get() ... maybe it somehow dangles? +static int cascade_fs_readdir(const char* path, void* buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info* fi, + enum fuse_readdir_flags flags) { + // TODO offsets for very large reads + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + // TODO fill stbuf properly? (check for get_stat errors) + // struct stat stbuf; + // memset(&stbuf, 0, sizeof(struct stat)); + + // FSTree::get_stat(node, &stbuf); + filler(buf, ".", nullptr, 0, (fuse_fill_dir_flags)0); + // TODO parent of root ??? + // FSTree::get_stat(node->parent, &stbuf); + filler(buf, "..", nullptr, 0, (fuse_fill_dir_flags)0); + for(const auto& [k, v] : node->children) { + if(filler(buf, k.c_str(), nullptr, 0, (fuse_fill_dir_flags)0)) { + break; + } + // FSTree::get_stat(v, &stbuf); + } + + return 0; +} + +static int cascade_fs_open(const char* path, struct fuse_file_info* fi) { + // TODO check O_ACCMODE, also check if dir ?? + auto node = fcc()->get(path); + if(fi->flags & O_CREAT && node == nullptr) { + node = fcc()->tree->add_op_key(path); + if(node == nullptr) { + return -ENOTSUP; + } + } + if(node == nullptr) { + return -ENOENT; + } + if(node->data.flag & OP_DIR) { + return -ENOTSUP; + } + if(fi->flags & O_TRUNC) { + node->data.bytes.clear(); + } + + fi->fh = reinterpret_cast(node); + return 0; +} + +static int cascade_fs_create(const char* path, mode_t mode, + struct fuse_file_info* fi) { + // TODO check mode ? currently ignores + return cascade_fs_open(path, fi); +} + +static int cascade_fs_read(const char* path, char* buf, size_t size, off_t offset, + struct fuse_file_info* fi) { + // auto node = reinterpret_cast(fi->fh); + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + if(node->data.flag & OP_DIR) { // TODO diff error + return -EACCES; + } + auto& bytes = node->data.bytes; + + size_t len = bytes.size(); + + if((size_t)offset < len) { + if(offset + size > len) { + size = len - offset; + } + auto p = reinterpret_cast(bytes.data()); + memcpy(buf, p + offset, size); + } else { + size = 0; + } + + return size; +} + +static int cascade_fs_write(const char* path, const char* buf, size_t size, + off_t offset, struct fuse_file_info* fi) { + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + if(node->data.flag & OP_DIR) { // TODO diff error + return -ENOTSUP; + } + auto& bytes = node->data.bytes; + + bytes.resize(std::max(bytes.size(), offset + size)); // TODO -ENOMEM + // TODO potentially undefined? + memcpy(bytes.data() + offset, buf, size); + + return size; +} + +/* +static int cascade_fs_flush(const char* path, struct fuse_file_info* fi) { + // TODO impl? (release is only called once at end while flush is called many times) + // There is no guarantee that flush will ever be called at all + return -ENOSYS; +} +*/ + +static int cascade_fs_release(const char* path, struct fuse_file_info* fi) { + auto node = fcc()->get(path); + if((fi->flags & O_ACCMODE) == O_RDONLY) { + return 0; + } + if(node == nullptr) { + return -ENOENT; + } + if(node->data.flag & OP_DIR) { + return -ENOTSUP; + } + return fcc()->put(node); + return 0; +} + +static int cascade_fs_mkdir(const char* path, mode_t mode) { + if(fcc()->get(path)) { + return -EEXIST; + } + auto op_root = fcc()->tree->object_pool_root(path); + if(op_root == nullptr) { + return -EACCES; + } + auto res = fcc()->tree->add_op_key_dir(path); + if(res) { + fcc()->local_dirs.insert(path); + // TODO err if nullptr + } + + return 0; +} + +static int cascade_fs_unlink(const char* path) { + return -ENOTSUP; + // TODO somehow create_object_pool says succeed on op root path (even tho it fails) + // TODO handle capi errors + // TODO for some reason remove on cascade client seg faults + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + if(node->data.flag & OP_DIR) { + return -EISDIR; + } + // TODO check open + + // remove + // auto result = capi.remove(key); + node->parent->children.erase(node->label); + delete node; + + return 0; +} + +static int cascade_fs_rmdir(const char* path) { + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + if(!(node->data.flag & OP_DIR)) { + return -ENOTDIR; + } + if(node->data.flag & (OP_PREFIX_DIR | OP_ROOT_DIR)) { + return -EACCES; + } + if(!node->children.empty()) { + return -ENOTEMPTY; + } + + // remove + fcc()->local_dirs.erase(path); + node->parent->children.erase(node->label); + delete node; + + return 0; +} + +static int cascade_fs_truncate(const char* path, off_t size, + struct fuse_file_info* fi) { + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + if(node->data.flag & OP_DIR) { + return -EINVAL; + } + node->data.bytes.clear(); + return fcc()->put(node); +} + +int set_buffer(char* dest, size_t size, const char* src, size_t len = 0) { + if(len == 0) { + len = strlen(src); + } + if(size == 0) { + return len; + } else if(size < len) { + return -ERANGE; + } + memcpy(dest, src, len); + return len; +} + +static int cascade_fs_chmod(const char* path, mode_t mode, struct fuse_file_info* fi) { + // TODO implement or -ENOTSUP + return 0; +} + +static int cascade_fs_chown(const char* path, uid_t uid, gid_t gid, struct fuse_file_info* fi) { + return -ENOTSUP; +} + +static int cascade_fs_utimens(const char*, const struct timespec tv[2], struct fuse_file_info* fi) { + // TODO implement or -ENOTSUP + return 0; +} + +#ifdef HAVE_SETXATTR +static int cascade_fs_setxattr(const char* path, const char* name, const char* value, + size_t size, int flags) { + // TODO save into node data + return -ENOTSUP; +} + +static int cascade_fs_getxattr(const char* path, const char* name, char* value, + size_t size) { + // TODO need to first apt get-install attr + if(strcmp(name, "user.cascade.version") == 0) { + return set_buffer(value, size, "CURRENT_VERSION"); + } else if(strcmp(name, "user.cascade.stable") == 0) { + return set_buffer(value, size, "0"); + } + return -ENODATA; +} + +static int cascade_fs_listxattr(const char* path, char* list, size_t size) { + // TODO lesson learned :(. returned 0 instead of length. check over all return types + // ^ 1hr+ bug + const char* names = "user.cascade.version\0" + "user.cascade.stable\0"; + // very weird behavior with \0 termination. strings and determining sizes did not work well + // ^ 2hr+ bug + return set_buffer(list, size, names, 41); +} +#endif + +// TODO create_object_pool, remove_object_pool, get_object_pool, op_remove +// op_get size. maybe no need to cache data? + +static const struct fuse_operations cascade_fs_oper = { + .getattr = cascade_fs_getattr, + // truncate, utimens, chown in place of setattr + .mkdir = cascade_fs_mkdir, + .unlink = cascade_fs_unlink, + .rmdir = cascade_fs_rmdir, + .chmod = cascade_fs_chmod, + .chown = cascade_fs_chown, + .truncate = cascade_fs_truncate, + .open = cascade_fs_open, + .read = cascade_fs_read, + .write = cascade_fs_write, + // .flush = cascade_fs_flush, + .release = cascade_fs_release, +#ifdef HAVE_SETXATTR + .setxattr = cascade_fs_setxattr, + .getxattr = cascade_fs_getxattr, + .listxattr = cascade_fs_listxattr, +#endif + .readdir = cascade_fs_readdir, + .init = cascade_fs_init, + .destroy = cascade_fs_destroy, + .create = cascade_fs_create, + .utimens = cascade_fs_utimens}; + +bool prepare_derecho_conf_file(const char* config_dir) { + // TODO for some reason needs to already be in correct directory ??? + // maybe derecho_conf_file envvar not correct? + // std::filesystem::path p(config_dir); + // const std::string conf_file = (p / "derecho.cfg").u8string(); + + // std::filesystem::current_path(config_dir); + const std::string conf_file = (std::filesystem::current_path() / "derecho.cfg").u8string(); + + struct stat buffer; + if(stat(conf_file.c_str(), &buffer) != 0) { + return false; + } + + const char* const derecho_conf_file = "DERECHO_CONF_FILE"; + setenv(derecho_conf_file, conf_file.c_str(), false); + dbg_default_info("Using derecho config file: {}.", getenv(derecho_conf_file)); + return true; +} + +int main(int argc, char* argv[]) { + options.client_dir = strdup("."); + + struct fuse_args args = FUSE_ARGS_INIT(argc, argv); + struct fuse* fuse = nullptr; + struct fuse_cmdline_opts opts; + struct fuse_loop_config config; + int res = 0; + + if(fuse_opt_parse(&args, &options, option_spec, nullptr) == -1) { + return 1; + } + + if(fuse_parse_cmdline(&args, &opts) != 0) { + return 1; + } + + try { + if(opts.show_version) { + printf("FUSE library version %s\n", fuse_pkgversion()); + fuse_lowlevel_version(); + res = 0; + throw 1; + } else if(opts.show_help) { + show_help(argv[0]); + fuse_cmdline_help(); + // fuse_lib_help(&args); + res = 0; + throw 1; + } else if(!opts.mountpoint) { + fprintf(stderr, "error: no mountpoint specified\n"); + res = 1; + throw 1; + } else if(!opts.singlethread) { + fprintf(stderr, "error: multi-threaded client not supported\n"); + res = 1; + throw 1; + } else if(!prepare_derecho_conf_file(options.client_dir)) { + fprintf(stderr, "error: invalid client directory\n" + "(dir needs derecho.cfg if DERECHO_CONF_FILE envvar is not set)\n"); + res = 1; + throw 1; + } + + try { + fuse = fuse_new(&args, &cascade_fs_oper, sizeof(cascade_fs_oper), nullptr); + if(fuse == nullptr) { + res = 1; + throw 1; + } + + if(fuse_mount(fuse, opts.mountpoint) != 0) { + res = 1; + throw 2; + } + + if(fuse_daemonize(opts.foreground) != 0) { + res = 1; + throw 3; + } + + struct fuse_session* se = fuse_get_session(fuse); + if(fuse_set_signal_handlers(se) != 0) { + res = 1; + throw 3; + } + if(opts.singlethread) { + res = fuse_loop(fuse); + } else { + config.clone_fd = opts.clone_fd; + config.max_idle_threads = opts.max_idle_threads; + res = fuse_loop_mt(fuse, &config); + } + if(res) { + res = 1; + } + + fuse_remove_signal_handlers(se); + } catch(derecho::derecho_exception& ex) { + std::cerr << "Exception:" << ex.what() << std::endl; + // TODO errors are not caught :( like cannot bind to socket + res = 1; + } + + throw 3; + } catch(int& ex) { + switch(ex) { + case 3: + fuse_unmount(fuse); + case 2: + fuse_destroy(fuse); + case 1: + free(opts.mountpoint); + fuse_opt_free_args(&args); + } + } + + return res; +} diff --git a/src/service/fuse/path_tree.hpp b/src/service/fuse/path_tree.hpp new file mode 100644 index 00000000..501b363b --- /dev/null +++ b/src/service/fuse/path_tree.hpp @@ -0,0 +1,158 @@ +#include +#include +namespace fs = std::filesystem; + +template +struct PathTree { + std::string label; + T data; + + PathTree* parent; + std::unordered_map*> children; + + PathTree(std::string label, T data, PathTree* parent) + : label(label), data(data), parent(parent) {} + PathTree(std::string label, T data) : PathTree(label, data, nullptr) {} + + ~PathTree() { + for(auto& [k, v] : children) { + delete v; + } + } + + std::vector entries() const { + std::vector res; + res.reserve(children.size()); + for(const auto& [k, _] : children) { + res.push_back(k); + } + return res; + } + + std::string absolute_path() const { + std::vector parts; + for(const PathTree* node = this; node != nullptr; node = node->parent) { + parts.push_back(node->label); + } + std::string res; + for(auto it = parts.rbegin(); it != parts.rend(); ++it) { + if(res.compare("") != 0 && res.compare("/") != 0) { + res += "/"; + } + res += *it; + } + + return res; + } + + void print(int depth = 0, std::ostream& stream = std::cout, + int pad = 0) const { + for(int i = 0; i < pad - 1; ++i) { + stream << " "; + } + if(pad) { + stream << "|-"; + } + stream << label << "\n"; + if(children.empty() || depth == 0) { + return; + } + for(const auto& [_, v] : children) { + v->print(depth - 1, stream, pad + 1); + } + if(pad == 0) { + stream << "\n\n"; + } + } + + // returns nullptr on fail or if location already exists + PathTree* set(const fs::path& path, T intermediate, T data) { + if(path.empty()) { + return nullptr; + } + auto it = path.begin(); + if(*it != label) { + return nullptr; + } + bool created_new = false; + PathTree* cur = this; + for(++it; it != path.end(); ++it) { + if(!cur->children.count(*it)) { + created_new = true; + PathTree* next = new PathTree(*it, intermediate, cur); + cur->children.insert({*it, next}); + cur = next; + } else { + cur = cur->children.at(*it); + } + } + if(!created_new) { + return nullptr; + } + cur->data = data; + return cur; + } + + // returns nullptr on fail or location does not exist + PathTree* get(const fs::path& path) { + if(path.empty()) { + return nullptr; + } + auto it = path.begin(); + if(*it != label) { + return nullptr; + } + PathTree* cur = this; + for(++it; it != path.end(); ++it) { + if(!cur->children.count(*it)) { + return nullptr; + } + cur = cur->children.at(*it); + } + return cur; + } + + PathTree* get_while_valid(const fs::path& path) { + if(path.empty()) { + return nullptr; + } + auto it = path.begin(); + if(*it != label) { + return nullptr; + } + PathTree* cur = this; + for(++it; it != path.end(); ++it) { + if(!cur->children.count(*it)) { + return cur; + } + cur = cur->children.at(*it); + } + return cur; + } + + PathTree* extract(const fs::path& path) { + PathTree* cur = get(path); + if(cur == nullptr || cur->parent == nullptr) { + return nullptr; + } + PathTree* par = cur->parent; + par->children.erase(cur->label); + + cur->parent = nullptr; + return cur; + } + + // deletes node and replaces with replacement. updating parent + // requires same label + static bool replace(PathTree* node, PathTree* replacement) { + if(node == nullptr || replacement == nullptr || node->label != replacement->label || node->parent == nullptr) { + return false; + } + std::cout << "replacing " << node->absolute_path() << std::endl; + PathTree* par = node->parent; + replacement->parent = par; + par->children[node->label] = replacement; + delete node; + return true; + } +}; From e7f8fb391dc31a6ace8ecbb7d10d30967218814b Mon Sep 17 00:00:00 2001 From: Michael Ye Date: Wed, 8 Feb 2023 23:48:06 -0500 Subject: [PATCH 07/10] feat: add fuse versioning --- include/cascade/object.hpp | 7 +- src/service/fuse/fcc_hl.hpp | 166 +++++++++++++++++----------- src/service/fuse/fuse_client_hl.cpp | 70 ++++++++---- 3 files changed, 155 insertions(+), 88 deletions(-) diff --git a/include/cascade/object.hpp b/include/cascade/object.hpp index 18fd9a38..a7b49e3f 100644 --- a/include/cascade/object.hpp +++ b/include/cascade/object.hpp @@ -211,7 +211,9 @@ class ObjectWithUInt64Key : public mutils::ByteRepresentable, }; inline std::ostream& operator<<(std::ostream& out, const Blob& b) { - out << "[size:" << b.size << ", data:" << std::hex; + out << "[size:" << b.size; + // out << ", data:"; + // out << std::quoted(reinterpret_cast(b.bytes)); // if(b.size > 0) { // uint32_t i = 0; // for(i = 0; i < 8 && i < b.size; i++) { @@ -221,8 +223,7 @@ inline std::ostream& operator<<(std::ostream& out, const Blob& b) { // out << "..."; // } // } - out << std::quoted(reinterpret_cast(b.bytes)); - out << std::dec << "]"; + out << "]"; return out; } diff --git a/src/service/fuse/fcc_hl.hpp b/src/service/fuse/fcc_hl.hpp index de05312f..3d92d260 100644 --- a/src/service/fuse/fcc_hl.hpp +++ b/src/service/fuse/fcc_hl.hpp @@ -13,8 +13,6 @@ namespace fs = std::filesystem; -struct timespec GLOBAL_INIT_TIMESTAMP; - std::shared_ptr DL; enum NodeFlag { @@ -29,28 +27,23 @@ const int OP_DIR = OP_PREFIX_DIR | OP_ROOT_DIR | OP_KEY_DIR; struct NodeData { NodeFlag flag; - // bool is_dir; - // bool is_op_prefix; - struct timespec init_timestamp; + // timestamp in microsec + uint64_t timestamp; std::vector bytes; - NodeData(const NodeFlag flag) : flag(flag) { - init_timestamp = GLOBAL_INIT_TIMESTAMP; - } - - NodeData(const NodeFlag flag, const std::string& contents) : flag(flag) { - init_timestamp = GLOBAL_INIT_TIMESTAMP; - bytes = std::vector(contents.begin(), contents.end()); + NodeData(const NodeFlag flag) : flag(flag), timestamp(0) { } }; +static const char* ROOT = "/"; + struct FSTree { using Node = PathTree; Node* root; FSTree() { - root = new Node("/", NodeData(OP_PREFIX_DIR)); + root = new Node(ROOT, NodeData(OP_PREFIX_DIR)); } ~FSTree() { delete root; } @@ -60,6 +53,15 @@ struct FSTree { return root->set(path, NodeData(OP_PREFIX_DIR), NodeData(OP_ROOT_DIR)); } + Node* add_op_info(const fs::path& path, const std::string& contents) { + auto node = root->set(path, NodeData(OP_PREFIX_DIR), NodeData(OP_INFO)); + if(node == nullptr) { + return nullptr; + } + node->data.bytes = std::vector(contents.begin(), contents.end()); + return node; + } + Node* add_op_key(const fs::path& path) { // invariant: assumes op_root already exists return root->set(path, NodeData(OP_KEY_DIR), NodeData(OP_KEY)); @@ -70,9 +72,9 @@ struct FSTree { return root->set(path, NodeData(OP_KEY_DIR), NodeData(OP_KEY_DIR)); } - void new_op_paths(const std::vector paths) { + void new_op_paths(const std::vector& paths) { auto old_root = root; - root = new Node("/", NodeData(OP_PREFIX_DIR)); + root = new Node(ROOT, NodeData(OP_PREFIX_DIR)); for(const auto& path : paths) { auto op_root = add_op_root(path); if(op_root == nullptr) { @@ -101,46 +103,20 @@ struct FSTree { auto node = root->get_while_valid(path); return object_pool_root(node); } - - static int get_stat(Node* node, struct stat* stbuf) { - if(node == nullptr) { - return -ENOENT; - } - // not needed: st_dev, st_blksize, st_ino (unless use_ino mount option) - // TODO maintain stbuf in data? - stbuf->st_nlink = 1; - stbuf->st_uid = fuse_get_context()->uid; - stbuf->st_gid = fuse_get_context()->gid; - stbuf->st_atim = node->data.init_timestamp; // TODO set properly - stbuf->st_mtim = node->data.init_timestamp; - stbuf->st_ctim = node->data.init_timestamp; - // - at prefix dir location add .info file ??? - - if(node->data.flag & OP_DIR) { - if(node->data.flag & OP_PREFIX_DIR) { - stbuf->st_mode = S_IFDIR | 0555; - } else { - stbuf->st_mode = S_IFDIR | 0755; - } - stbuf->st_nlink = 2; // TODO calculate properly - for(const auto& [_, v] : node->children) { - stbuf->st_nlink += v->data.flag & OP_DIR; - } - } else { - // TODO somehow even when 0444, can still write ??? - stbuf->st_mode = S_IFREG | 0744; - stbuf->st_size = node->data.bytes.size(); - } - return 0; - } }; namespace derecho { namespace cascade { struct FuseClientContext { + const std::string INFO_EXT = ".cacade"; ServiceClientAPI& capi; + persistent::version_t ver; + // TODO + persistent::version_t max_ver; + bool latest; + FSTree* tree; std::set local_dirs; @@ -152,7 +128,10 @@ struct FuseClientContext { DL = LoggerFactory::createLogger("fuse_client", spdlog::level::from_str(derecho::getConfString(CONF_LOGGER_DEFAULT_LOG_LEVEL))); DL->set_pattern("[%T][%n][%^%l%$] %v"); - clock_gettime(CLOCK_REALTIME, &GLOBAL_INIT_TIMESTAMP); + // clock_gettime(CLOCK_REALTIME, &GLOBAL_INIT_TIMESTAMP); + + ver = 0; + latest = true; update_interval = 15; last_update_sec = 0; @@ -199,19 +178,38 @@ struct FuseClientContext { } void update_object_pools() { - auto reply = capi.list_object_pools(true); - std::vector paths(reply.size()); - for(size_t i = 0; i < reply.size(); ++i) { - paths[i] = fs::path(reply[i]); - } - // TODO avoid re generating op keys? ... by not resetting tree FSTree* old_tree = tree; delete old_tree; tree = new FSTree(); + // TODO no version for list object pools + auto reply = capi.list_object_pools(true); + std::vector paths(reply.size()); + for(size_t i = 0; i < reply.size(); ++i) { + paths[i] = fs::path(reply[i]); + } tree->new_op_paths(paths); + for(size_t i = 0; i < reply.size(); ++i) { + auto opm = capi.find_object_pool(reply[i]); + json j{ + {"valid", opm.is_valid()}, + {"null", opm.is_null()}}; + if(opm.is_valid() && !opm.is_null()) { + j.emplace("pathname", opm.pathname); + j.emplace("version", opm.version); + j.emplace("timestamp_us", opm.timestamp_us); + j.emplace("previous_version", opm.previous_version); + j.emplace("previous_version_by_key", opm.previous_version_by_key); + j.emplace("subgroup_type", std::to_string(opm.subgroup_type_index) + "-->" + DefaultObjectPoolMetadataType::subgroup_type_order[opm.subgroup_type_index].name()); + j.emplace("subgroup_index", opm.subgroup_index); + j.emplace("sharding_policy", std::to_string(opm.sharding_policy)); + j.emplace("deleted", opm.deleted); + } + tree->add_op_info(reply[i] + INFO_EXT, j.dump(2)); + } + for(const auto& op_root : paths) { if(tree->root->get(op_root) != nullptr) { auto keys = get_keys(op_root); @@ -222,7 +220,7 @@ struct FuseClientContext { if(key == nullptr) { dbg_info(DL, "did not add {}", k); } else { - key->data.bytes = get_contents(k); + get_contents(key, k); // dbg_info(DL, "file: {}", std::quoted(reinterpret_cast(key->data.bytes.data()))); } } @@ -244,35 +242,75 @@ struct FuseClientContext { last_update_sec = time(0); } + int get_stat(FSTree::Node* node, struct stat* stbuf) { + if(node == nullptr) { + return -ENOENT; + } + // not needed: st_dev, st_blksize, st_ino (unless use_ino mount option) + // TODO maintain stbuf in data? + stbuf->st_nlink = 1; + stbuf->st_uid = fuse_get_context()->uid; + stbuf->st_gid = fuse_get_context()->gid; + stbuf->st_atim = timespec{last_update_sec, 0}; + // TODO merge timestamp for dirs, update during set + int64_t sec = node->data.timestamp / 1'000'000; + int64_t nano = (node->data.timestamp % 1'000'000) * 1000; + stbuf->st_mtim = timespec{sec, nano}; + stbuf->st_ctim = stbuf->st_mtim; + // - at prefix dir location add .info file ??? + + if(node->data.flag & OP_DIR) { + if(node->data.flag & OP_PREFIX_DIR) { + stbuf->st_mode = S_IFDIR | 0555; + } else { + stbuf->st_mode = S_IFDIR | 0755; + } + stbuf->st_nlink = 2; // TODO calculate properly + for(const auto& [_, v] : node->children) { + stbuf->st_nlink += v->data.flag & OP_DIR; + } + } else { + // TODO somehow even when 0444, can still write ??? + stbuf->st_mode = S_IFREG | (node->data.flag & OP_KEY ? 0744 : 0444); + stbuf->st_size = node->data.bytes.size(); + } + return 0; + } + std::string string() { std::stringstream ss; tree->root->print(100, ss); return ss.str(); } - std::vector get_contents(const std::string& path) { - persistent::version_t version = CURRENT_VERSION; - auto result = capi.get(path, version, true); + void get_contents(FSTree::Node* node, const std::string& path) { + // persistent::version_t version = CURRENT_VERSION; + auto result = capi.get(path, latest ? CURRENT_VERSION : ver, true); + // TODO only get file contents on open for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); + if(latest) { + ver = std::max(ver, reply.version); + } Blob blob = reply.blob; std::vector bytes(blob.bytes, blob.bytes + blob.size); - // TODO!!! don't use .data() - // memcpy(bytes.data(), blob.bytes, blob.size); - return bytes; + node->data.bytes = bytes; + node->data.timestamp = reply.timestamp_us; + return; } - return {}; } std::vector get_keys(const std::string& path) { - persistent::version_t version = CURRENT_VERSION; - auto future_result = capi.list_keys(version, true, path); + // persistent::version_t version = CURRENT_VERSION; + auto future_result = capi.list_keys(latest ? CURRENT_VERSION : ver, true, path); auto reply = capi.wait_list_keys(future_result); // TODO remove keys that collide with directory (key is a prefix of another key) return reply; } + // make a trash folder? (move on delete) + FSTree::Node* get(const std::string& path) { if(should_update()) { update_object_pools(); diff --git a/src/service/fuse/fuse_client_hl.cpp b/src/service/fuse/fuse_client_hl.cpp index 3b64c6c8..02aa2c54 100644 --- a/src/service/fuse/fuse_client_hl.cpp +++ b/src/service/fuse/fuse_client_hl.cpp @@ -1,3 +1,5 @@ +#define HAVE_SETXATTR + #include #include #include @@ -12,11 +14,12 @@ #include "fcc_hl.hpp" #include "fuse_client_signals.hpp" #include +#ifdef HAVE_SETXATTR +#include +#endif using namespace derecho::cascade; -#define HAVE_SETXATTR - struct cli_options { const char* client_dir; int update_interval; @@ -70,7 +73,7 @@ static int cascade_fs_getattr(const char* path, struct stat* stbuf, return -ENOENT; } memset(stbuf, 0, sizeof(struct stat)); - return FSTree::get_stat(node, stbuf); + return fcc()->get_stat(node, stbuf); } // TODO :( invalid pointer dumped?? somehow cascade replys needs to be stored in a variable before @@ -217,10 +220,7 @@ static int cascade_fs_mkdir(const char* path, mode_t mode) { } static int cascade_fs_unlink(const char* path) { - return -ENOTSUP; - // TODO somehow create_object_pool says succeed on op root path (even tho it fails) - // TODO handle capi errors - // TODO for some reason remove on cascade client seg faults + // return -ENOTSUP; auto node = fcc()->get(path); if(node == nullptr) { return -ENOENT; @@ -231,9 +231,11 @@ static int cascade_fs_unlink(const char* path) { // TODO check open // remove + node->data.bytes.clear(); + // auto result = capi.remove(key); - node->parent->children.erase(node->label); - delete node; + // node->parent->children.erase(node->label); + // delete node; return 0; } @@ -270,7 +272,7 @@ static int cascade_fs_truncate(const char* path, off_t size, if(node->data.flag & OP_DIR) { return -EINVAL; } - node->data.bytes.clear(); + node->data.bytes.resize(size, 0); return fcc()->put(node); } @@ -304,34 +306,60 @@ static int cascade_fs_utimens(const char*, const struct timespec tv[2], struct f #ifdef HAVE_SETXATTR static int cascade_fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags) { + if(flags & XATTR_CREATE) { + return -EPERM; + } + dbg_info(DL, "{}, {}, {}", path, name, value); // TODO save into node data - return -ENOTSUP; + if(strcmp(path, ROOT) == 0) { + if(!fcc()->latest && strcmp(name, "user.cascade.version") == 0) { + fcc()->ver = strtoll(value, nullptr, 0); + // TODO + fcc()->update_object_pools(); + return 0; + } else if(strcmp(name, "user.cascade.latest") == 0) { + fcc()->latest = strcmp(value, "1") == 0; + fcc()->update_object_pools(); + return 0; + } + } + return -EPERM; } static int cascade_fs_getxattr(const char* path, const char* name, char* value, size_t size) { // TODO need to first apt get-install attr - if(strcmp(name, "user.cascade.version") == 0) { - return set_buffer(value, size, "CURRENT_VERSION"); - } else if(strcmp(name, "user.cascade.stable") == 0) { - return set_buffer(value, size, "0"); + if(strcmp(path, ROOT) == 0) { + if(strcmp(name, "user.cascade.version") == 0) { + std::string v = std::to_string(fcc()->ver); + return set_buffer(value, size, v.c_str()); + } + if(strcmp(name, "user.cascade.latest") == 0) { + return set_buffer(value, size, fcc()->latest ? "1" : "0"); + } } return -ENODATA; } static int cascade_fs_listxattr(const char* path, char* list, size_t size) { // TODO lesson learned :(. returned 0 instead of length. check over all return types - // ^ 1hr+ bug - const char* names = "user.cascade.version\0" - "user.cascade.stable\0"; - // very weird behavior with \0 termination. strings and determining sizes did not work well - // ^ 2hr+ bug - return set_buffer(list, size, names, 41); + if(strcmp(path, ROOT) == 0) { + // ^ 1hr+ bug + const char names[] = "user.cascade.version\0" + "user.cascade.latest\0"; + // very weird behavior with \0 termination. strings and determining sizes did not work well + // ^ 2hr+ bug + return set_buffer(list, size, names, 41); + } + const char empty[] = ""; + return set_buffer(list, size, empty, 0); } #endif // TODO create_object_pool, remove_object_pool, get_object_pool, op_remove // op_get size. maybe no need to cache data? +// TODO version/#/... (mount) +// maybe use attr static const struct fuse_operations cascade_fs_oper = { .getattr = cascade_fs_getattr, From ee3829ab157e1d34b792adb85409e6a20cc72572 Mon Sep 17 00:00:00 2001 From: Michael Ye Date: Thu, 16 Feb 2023 13:17:05 -0500 Subject: [PATCH 08/10] fix: fuse change to use max_ver --- scripts/util/cascade_runner.sh | 2 +- src/service/fuse/fcc_hl.hpp | 3 ++- src/service/fuse/fuse_client_hl.cpp | 16 +++++++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/util/cascade_runner.sh b/scripts/util/cascade_runner.sh index 7c57c4fa..220f9e41 100755 --- a/scripts/util/cascade_runner.sh +++ b/scripts/util/cascade_runner.sh @@ -39,7 +39,7 @@ BASE_PATH="/root/workspace/cascade/build-Release/src" export PATH="$PATH:$BASE_PATH/service" export PATH="$PATH:$BASE_PATH/service/fuse" -FUSE_COMMAND="cascade_fuse_client" +FUSE_COMMAND="cascade_fuse_client_hl" # "cascade_fuse_client" CLIENT_COMMAND="cascade_client" SERVER_COMMAND="cascade_server" LOG_FILE="node_info.log" diff --git a/src/service/fuse/fcc_hl.hpp b/src/service/fuse/fcc_hl.hpp index 3d92d260..f30d7f64 100644 --- a/src/service/fuse/fcc_hl.hpp +++ b/src/service/fuse/fcc_hl.hpp @@ -131,6 +131,7 @@ struct FuseClientContext { // clock_gettime(CLOCK_REALTIME, &GLOBAL_INIT_TIMESTAMP); ver = 0; + max_ver = 0; latest = true; update_interval = 15; @@ -291,7 +292,7 @@ struct FuseClientContext { for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); if(latest) { - ver = std::max(ver, reply.version); + max_ver = std::max(max_ver, reply.version); } Blob blob = reply.blob; std::vector bytes(blob.bytes, blob.bytes + blob.size); diff --git a/src/service/fuse/fuse_client_hl.cpp b/src/service/fuse/fuse_client_hl.cpp index 02aa2c54..dcce9bce 100644 --- a/src/service/fuse/fuse_client_hl.cpp +++ b/src/service/fuse/fuse_client_hl.cpp @@ -333,9 +333,11 @@ static int cascade_fs_getxattr(const char* path, const char* name, char* value, if(strcmp(name, "user.cascade.version") == 0) { std::string v = std::to_string(fcc()->ver); return set_buffer(value, size, v.c_str()); - } - if(strcmp(name, "user.cascade.latest") == 0) { + } else if(strcmp(name, "user.cascade.latest") == 0) { return set_buffer(value, size, fcc()->latest ? "1" : "0"); + } else if(strcmp(name, "user.cascade.largest_known_version") == 0) { + std::string v = std::to_string(fcc()->max_ver); + return set_buffer(value, size, v.c_str()); } } return -ENODATA; @@ -345,11 +347,15 @@ static int cascade_fs_listxattr(const char* path, char* list, size_t size) { // TODO lesson learned :(. returned 0 instead of length. check over all return types if(strcmp(path, ROOT) == 0) { // ^ 1hr+ bug - const char names[] = "user.cascade.version\0" - "user.cascade.latest\0"; + const char names[] = "user.cascade.largest_known_version" + "\0" + "user.cascade.version" + "\0" + "user.cascade.latest" + "\0"; // very weird behavior with \0 termination. strings and determining sizes did not work well // ^ 2hr+ bug - return set_buffer(list, size, names, 41); + return set_buffer(list, size, names, sizeof(names) / sizeof(names[0]) - 1); } const char empty[] = ""; return set_buffer(list, size, empty, 0); From 8b72789a68c47a629b129a3e6c7d2089c0454e6a Mon Sep 17 00:00:00 2001 From: Michael Ye Date: Thu, 2 Mar 2023 15:32:29 -0500 Subject: [PATCH 09/10] switch to latest and snapshot folders --- src/service/fuse/fcc_hl.hpp | 346 +++++++++++++---------- src/service/fuse/fuse_client_context.hpp | 71 ++--- src/service/fuse/fuse_client_hl.cpp | 65 +++-- src/service/fuse/path_tree.hpp | 1 + 4 files changed, 246 insertions(+), 237 deletions(-) diff --git a/src/service/fuse/fcc_hl.hpp b/src/service/fuse/fcc_hl.hpp index f30d7f64..2ccfdcde 100644 --- a/src/service/fuse/fcc_hl.hpp +++ b/src/service/fuse/fcc_hl.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -15,15 +16,27 @@ namespace fs = std::filesystem; std::shared_ptr DL; -enum NodeFlag { - OP_PREFIX_DIR = 1 << 0, - OP_ROOT_DIR = 1 << 1, - OP_KEY_DIR = 1 << 2, - OP_KEY = 1 << 3, - OP_INFO = 1 << 4, +enum NodeFlag : uint32_t { + ROOT_DIR = 1 << 0, + + OP_PREFIX_DIR = 1 << 1, + OP_ROOT_DIR = 1 << 2, + + KEY_DIR = 1 << 3, + KEY_FILE = 1 << 4, + + LATEST_DIR = 1 << 5, + + METADATA_PREFIX_DIR = 1 << 6, + METADATA_INFO_FILE = 1 << 7, + + SNAPSHOT_ROOT_DIR = 1 << 8, + SNAPSHOT_TIME_DIR = 1 << 9, }; -const int OP_DIR = OP_PREFIX_DIR | OP_ROOT_DIR | OP_KEY_DIR; +const uint32_t FILE_FLAG = KEY_FILE | METADATA_INFO_FILE; +const uint32_t DIR_FLAG = ~FILE_FLAG; +const uint32_t OP_FLAG = OP_PREFIX_DIR | OP_ROOT_DIR | KEY_DIR | KEY_FILE; struct NodeData { NodeFlag flag; @@ -35,26 +48,60 @@ struct NodeData { } }; -static const char* ROOT = "/"; +namespace derecho { +namespace cascade { -struct FSTree { +struct FuseClientContext { using Node = PathTree; - Node* root; + std::unique_ptr root; + + // object pools are not versioned? :( + const fs::path METADATA_PATH = "/.cascade"; + const fs::path SNAPSHOT_PATH = "/snapshot"; + const fs::path LATEST_PATH = "/latest"; + const fs::path ROOT = "/"; + ServiceClientAPI& capi; + + persistent::version_t max_ver; + + std::set local_latest_dirs; + std::set snapshots; + + time_t update_interval; + time_t last_update_sec; + + FuseClientContext() : capi(ServiceClientAPI::get_service_client()) { + DL = LoggerFactory::createLogger("fuse_client", spdlog::level::from_str(derecho::getConfString(CONF_LOGGER_DEFAULT_LOG_LEVEL))); + DL->set_pattern("[%T][%n][%^%l%$] %v"); + + max_ver = 0; + + update_interval = 15; + last_update_sec = 0; + + root = std::make_unique(ROOT, NodeData(ROOT_DIR)); + root->set(SNAPSHOT_PATH, NodeData(SNAPSHOT_ROOT_DIR), NodeData(SNAPSHOT_ROOT_DIR)); + reset_latest(); - FSTree() { - root = new Node(ROOT, NodeData(OP_PREFIX_DIR)); + update_object_pools(); } - ~FSTree() { delete root; } + /* --- pathtree related logic --- */ - Node* add_op_root(const fs::path& path) { - // TODO add info - return root->set(path, NodeData(OP_PREFIX_DIR), NodeData(OP_ROOT_DIR)); + void reset_latest() { + auto latest = root->get(LATEST_PATH); + if(latest != nullptr) { + latest->data = NodeData(LATEST_DIR); + latest->children.clear(); + } else { + root->set(LATEST_PATH, NodeData(LATEST_DIR), NodeData(LATEST_DIR)); + } } Node* add_op_info(const fs::path& path, const std::string& contents) { - auto node = root->set(path, NodeData(OP_PREFIX_DIR), NodeData(OP_INFO)); + auto node = root->set(path, NodeData(METADATA_PREFIX_DIR), + NodeData(METADATA_INFO_FILE)); if(node == nullptr) { return nullptr; } @@ -62,95 +109,57 @@ struct FSTree { return node; } + Node* add_snapshot_time(const fs::path& path) { + return root->set(path, NodeData(SNAPSHOT_ROOT_DIR), NodeData(SNAPSHOT_TIME_DIR)); + } + + Node* add_op_root(const fs::path& path) { + return root->set(path, NodeData(OP_PREFIX_DIR), NodeData(OP_ROOT_DIR)); + } + Node* add_op_key(const fs::path& path) { // invariant: assumes op_root already exists - return root->set(path, NodeData(OP_KEY_DIR), NodeData(OP_KEY)); + return root->set(path, NodeData(KEY_DIR), NodeData(KEY_FILE)); } Node* add_op_key_dir(const fs::path& path) { // invariant: assumes op_root already exists - return root->set(path, NodeData(OP_KEY_DIR), NodeData(OP_KEY_DIR)); - } - - void new_op_paths(const std::vector& paths) { - auto old_root = root; - root = new Node(ROOT, NodeData(OP_PREFIX_DIR)); - for(const auto& path : paths) { - auto op_root = add_op_root(path); - if(op_root == nullptr) { - continue; - // TODO errors - } - auto old_op_root = old_root->extract(path); - if(old_op_root != nullptr) { - Node::replace(op_root, old_op_root); - } - } - delete old_root; + return root->set(path, NodeData(KEY_DIR), NodeData(KEY_DIR)); } - static Node* object_pool_root(Node* node) { - if(node == nullptr || node->data.flag & OP_PREFIX_DIR) { + Node* object_pool_root(Node* node) { + if(node == nullptr) { return nullptr; } if(node->parent == nullptr || node->data.flag & OP_ROOT_DIR) { return node; } - return object_pool_root(node->parent); + if(node->data.flag & (KEY_DIR | KEY_FILE)) { + return object_pool_root(node->parent); + } + return nullptr; } - Node* object_pool_root(const fs::path& path) { + Node* nearest_object_pool_root(const fs::path& path) { auto node = root->get_while_valid(path); return object_pool_root(node); } -}; - -namespace derecho { -namespace cascade { - -struct FuseClientContext { - const std::string INFO_EXT = ".cacade"; - ServiceClientAPI& capi; - - persistent::version_t ver; - // TODO - persistent::version_t max_ver; - bool latest; - - FSTree* tree; - - std::set local_dirs; - - time_t update_interval; - time_t last_update_sec; - - FuseClientContext() : capi(ServiceClientAPI::get_service_client()) { - DL = LoggerFactory::createLogger("fuse_client", spdlog::level::from_str(derecho::getConfString(CONF_LOGGER_DEFAULT_LOG_LEVEL))); - DL->set_pattern("[%T][%n][%^%l%$] %v"); - - // clock_gettime(CLOCK_REALTIME, &GLOBAL_INIT_TIMESTAMP); - - ver = 0; - max_ver = 0; - latest = true; - - update_interval = 15; - last_update_sec = 0; - - tree = new FSTree(); - - update_object_pools(); - } - ~FuseClientContext() { + persistent::version_t is_snapshot(const fs::path& path) { + if(path.parent_path() != SNAPSHOT_PATH) { + return CURRENT_VERSION; + } try { - dbg_debug(DL, "deleting tree"); - delete tree; - } catch(...) { - dbg_error(DL, "could not delete tree"); + return std::stoull(path.filename()); + } catch(std::invalid_argument const& ex) { + return CURRENT_VERSION; + } catch(std::out_of_range const& ex) { + return CURRENT_VERSION; } } + /* --- capi related logic --- */ + bool should_update() { if(time(0) > last_update_sec + update_interval) { return true; @@ -158,10 +167,23 @@ struct FuseClientContext { return false; } - int put(const FSTree::Node* node) { + std::string path_while_op(const Node* node) const { + std::vector parts; + for(; node != nullptr && (node->data.flag & OP_FLAG); node = node->parent) { + parts.push_back(node->label); + } + std::string res; + for(auto it = parts.rbegin(); it != parts.rend(); ++it) { + res += "/" + *it; + } + + return res; + } + + int put_to_capi(const Node* node) { // invariant: node is file ObjectWithStringKey obj; - obj.key = node->absolute_path(); + obj.key = path_while_op(node); obj.previous_version = INVALID_VERSION; obj.previous_version_by_key = INVALID_VERSION; obj.blob = Blob(node->data.bytes.data(), node->data.bytes.size(), true); @@ -178,25 +200,25 @@ struct FuseClientContext { return 0; } - void update_object_pools() { - // TODO avoid re generating op keys? ... by not resetting tree - FSTree* old_tree = tree; - delete old_tree; - tree = new FSTree(); - + void fill_at(const fs::path& prefix, persistent::version_t ver) { // TODO no version for list object pools - auto reply = capi.list_object_pools(true); - std::vector paths(reply.size()); - for(size_t i = 0; i < reply.size(); ++i) { - paths[i] = fs::path(reply[i]); - } - tree->new_op_paths(paths); + auto str_paths = capi.list_object_pools(false, true); + + auto meta_path = prefix; + meta_path += METADATA_PATH; + root->set(meta_path, + NodeData(METADATA_PREFIX_DIR), NodeData(METADATA_PREFIX_DIR)); + for(const std::string& op_root : str_paths) { + auto op_root_path = prefix; + op_root_path += op_root; + auto op_root_node = add_op_root(op_root_path); + if(op_root_node == nullptr) { + continue; + } - for(size_t i = 0; i < reply.size(); ++i) { - auto opm = capi.find_object_pool(reply[i]); - json j{ - {"valid", opm.is_valid()}, - {"null", opm.is_null()}}; + auto opm = capi.find_object_pool(op_root); + json j{{"valid", opm.is_valid()}, + {"null", opm.is_null()}}; if(opm.is_valid() && !opm.is_null()) { j.emplace("pathname", opm.pathname); j.emplace("version", opm.version); @@ -208,32 +230,52 @@ struct FuseClientContext { j.emplace("sharding_policy", std::to_string(opm.sharding_policy)); j.emplace("deleted", opm.deleted); } - tree->add_op_info(reply[i] + INFO_EXT, j.dump(2)); - } - - for(const auto& op_root : paths) { - if(tree->root->get(op_root) != nullptr) { - auto keys = get_keys(op_root); - std::sort(keys.begin(), keys.end(), std::greater<>()); - for(const auto& k : keys) { - // TODO verify remove file colliding with directory here - auto key = tree->add_op_key(k); - if(key == nullptr) { - dbg_info(DL, "did not add {}", k); - } else { - get_contents(key, k); - // dbg_info(DL, "file: {}", std::quoted(reinterpret_cast(key->data.bytes.data()))); - } + auto op_root_meta_path = prefix; + op_root_meta_path += METADATA_PATH; + op_root_meta_path += op_root; + add_op_info(op_root_meta_path, j.dump(2)); + + auto keys = get_keys(op_root, ver); + std::sort(keys.begin(), keys.end(), std::greater<>()); + // sort removes files colliding with directory + for(const auto& k : keys) { + auto key_path = prefix; + key_path += k; + auto node = add_op_key(key_path); + if(node == nullptr) { + dbg_info(DL, "did not add {}", key_path); + } else { + get_contents(node, k, ver); + // dbg_info(DL, "file: {}", std::quoted(reinterpret_cast(node->data.bytes.data()))); } } } + } + + void add_snapshot(persistent::version_t ver) { + auto snapshot = SNAPSHOT_PATH; + snapshot += "/" + std::to_string(ver); + auto res = add_snapshot_time(snapshot); + dbg_info(DL, "adding {}", snapshot); + if(res != nullptr) { + dbg_info(DL, "ok"); + fill_at(snapshot, ver); + dbg_info(DL, "filled"); + } + } + + void update_object_pools() { + // TODO use old cached data + reset_latest(); - for(auto it = local_dirs.begin(); it != local_dirs.end();) { - if(tree->object_pool_root(*it) == nullptr) { + fill_at(LATEST_PATH, CURRENT_VERSION); + + for(auto it = local_latest_dirs.begin(); it != local_latest_dirs.end();) { + if(nearest_object_pool_root(*it) == nullptr) { // TODO verify for op_root deleted - it = local_dirs.erase(it); + it = local_latest_dirs.erase(it); } else { - tree->add_op_key_dir(*it); + add_op_key_dir(*it); ++it; } } @@ -243,7 +285,7 @@ struct FuseClientContext { last_update_sec = time(0); } - int get_stat(FSTree::Node* node, struct stat* stbuf) { + int get_stat(Node* node, struct stat* stbuf) { if(node == nullptr) { return -ENOENT; } @@ -260,7 +302,7 @@ struct FuseClientContext { stbuf->st_ctim = stbuf->st_mtim; // - at prefix dir location add .info file ??? - if(node->data.flag & OP_DIR) { + if(node->data.flag & DIR_FLAG) { if(node->data.flag & OP_PREFIX_DIR) { stbuf->st_mode = S_IFDIR | 0555; } else { @@ -268,30 +310,45 @@ struct FuseClientContext { } stbuf->st_nlink = 2; // TODO calculate properly for(const auto& [_, v] : node->children) { - stbuf->st_nlink += v->data.flag & OP_DIR; + stbuf->st_nlink += v->data.flag & DIR_FLAG; } } else { // TODO somehow even when 0444, can still write ??? - stbuf->st_mode = S_IFREG | (node->data.flag & OP_KEY ? 0744 : 0444); + stbuf->st_mode = S_IFREG | (node->data.flag & KEY_FILE ? 0744 : 0444); stbuf->st_size = node->data.bytes.size(); } + + // dev_t st_dev; /* ID of device containing file */ + // ino_t st_ino; /* Inode number */ + // mode_t st_mode; /* File type and mode */ + // nlink_t st_nlink; /* Number of hard links */ + // uid_t st_uid; /* User ID of owner */ + // gid_t st_gid; /* Group ID of owner */ + // dev_t st_rdev; /* Device ID (if special file) */ + // off_t st_size; /* Total size, in bytes */ + // blksize_t st_blksize; /* Block size for filesystem I/O */ + // blkcnt_t st_blocks; /* Number of 512B blocks allocated */ + + // struct timespec st_atim; /* Time of last access */ + // struct timespec st_mtim; /* Time of last modification */ + // struct timespec st_ctim; /* Time of last status change */ return 0; } std::string string() { std::stringstream ss; - tree->root->print(100, ss); + root->print(100, ss); return ss.str(); } - void get_contents(FSTree::Node* node, const std::string& path) { + void get_contents(Node* node, const std::string& path, persistent::version_t ver) { // persistent::version_t version = CURRENT_VERSION; - auto result = capi.get(path, latest ? CURRENT_VERSION : ver, true); + auto result = capi.get(path, ver, true); // TODO only get file contents on open for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); - if(latest) { + if(ver == CURRENT_VERSION) { max_ver = std::max(max_ver, reply.version); } Blob blob = reply.blob; @@ -302,9 +359,9 @@ struct FuseClientContext { } } - std::vector get_keys(const std::string& path) { + std::vector get_keys(const std::string& path, persistent::version_t ver) { // persistent::version_t version = CURRENT_VERSION; - auto future_result = capi.list_keys(latest ? CURRENT_VERSION : ver, true, path); + auto future_result = capi.list_keys(ver, true, path); auto reply = capi.wait_list_keys(future_result); // TODO remove keys that collide with directory (key is a prefix of another key) return reply; @@ -312,38 +369,19 @@ struct FuseClientContext { // make a trash folder? (move on delete) - FSTree::Node* get(const std::string& path) { + Node* get(const std::string& path) { if(should_update()) { update_object_pools(); // split between update object pool list and update keys } - return tree->root->get(path); + return root->get(path); } // TODO use object pool root meta file to edit version # and such? // TODO cascade metaservice api. need: // - get children given path. shouldnt be hard considering op_list_keys works from subdir of op root // - get file metadata WITHOUT getting file contents (file size, modification time) - - /* - stat - - */ - // dev_t st_dev; /* ID of device containing file */ - // ino_t st_ino; /* Inode number */ - // mode_t st_mode; /* File type and mode */ - // nlink_t st_nlink; /* Number of hard links */ - // uid_t st_uid; /* User ID of owner */ - // gid_t st_gid; /* Group ID of owner */ - // dev_t st_rdev; /* Device ID (if special file) */ - // off_t st_size; /* Total size, in bytes */ - // blksize_t st_blksize; /* Block size for filesystem I/O */ - // blkcnt_t st_blocks; /* Number of 512B blocks allocated */ - - // struct timespec st_atim; /* Time of last access */ - // struct timespec st_mtim; /* Time of last modification */ - // struct timespec st_ctim; /* Time of last status change */ }; } // namespace cascade -} // namespace derecho \ No newline at end of file +} // namespace derecho diff --git a/src/service/fuse/fuse_client_context.hpp b/src/service/fuse/fuse_client_context.hpp index 7d47c281..29498d60 100644 --- a/src/service/fuse/fuse_client_context.hpp +++ b/src/service/fuse/fuse_client_context.hpp @@ -536,21 +536,6 @@ class KeyINode : public FuseDataINode { dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } - virtual uint64_t read_file(FileBytes* _file_bytes) override { - dbg_default_trace("[{}]entering {}.", gettid(), __func__); - check_update(); - _file_bytes->size = this->file_bytes.get()->size; - _file_bytes->bytes = static_cast(malloc(this->file_bytes.get()->size)); - memcpy(_file_bytes->bytes, this->file_bytes.get()->bytes, this->file_bytes.get()->size); - dbg_default_trace("[{}]leaving {}.", gettid(), __func__); - return 0; - } - - virtual uint64_t get_file_size() override { - check_update(); - return this->file_bytes.get()->size; - } - KeyINode(KeyINode&& fci) { this->type = std::move(fci.type); this->display_name = std::move(fci.display_name); @@ -559,12 +544,6 @@ class KeyINode : public FuseDataINode { this->capi = std::move(fci.capi); } - virtual ~KeyINode() { - dbg_default_info("[{}] entering {}.", gettid(), __func__); - file_bytes.reset(nullptr); - dbg_default_info("[{}] leaving {}.", gettid(), __func__); - } - private: virtual void update_contents() override { ShardINode* pino_shard = reinterpret_cast*>(this->parent); @@ -590,14 +569,19 @@ class KeyINode : public FuseDataINode { this->previous_version_by_key = access_ptr->previous_version_by_key; blob = access_ptr->blob; } else { - this->file_bytes.get()->size = mutils::bytes_size(reply); - this->file_bytes.get()->bytes = static_cast(malloc(this->file_bytes.get()->size)); - reply.to_bytes(this->file_bytes.get()->bytes); + // this->file_bytes.get()->size = mutils::bytes_size(reply); + // this->file_bytes.get()->bytes = static_cast(malloc(this->file_bytes.get()->size)); + // reply.to_bytes(this->file_bytes.get()->bytes); + this->cached_data.bytes.resize(mutils::bytes_size(reply)); + reply.to_bytes(this->cached_data.bytes.data()); return; } - file_bytes.get()->size = blob.size; - file_bytes.get()->bytes = static_cast(malloc(file_bytes.get()->size)); - memcpy(file_bytes.get()->bytes, blob.bytes, file_bytes.get()->size); + // file_bytes.get()->size = blob.size; + // file_bytes.get()->bytes = static_cast(malloc(file_bytes.get()->size)); + // memcpy(file_bytes.get()->bytes, blob.bytes, file_bytes.get()->size); + this->cached_data.bytes.resize(blob.size); + memcpy(this->cached_data.bytes.data(), blob.bytes, blob.size); + return; return; } } @@ -883,6 +867,7 @@ class ObjectPoolPathINode : public FuseClientINode { ++it; } } + return; } virtual std::map get_dir_entries() override { @@ -965,28 +950,6 @@ class ObjectPoolKeyINode : public FuseDataINode { dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } - virtual uint64_t read_file(FileBytes* _file_bytes) override { - dbg_default_debug("-- READ FILE of key:[{}], [{}]entering {}.", this->key, gettid(), __func__); - check_update(); - _file_bytes->size = this->file_bytes.get()->size; - _file_bytes->bytes = static_cast(malloc(this->file_bytes.get()->size)); - memcpy(_file_bytes->bytes, this->file_bytes.get()->bytes, this->file_bytes.get()->size); - dbg_default_debug("[{}]leaving {}.", gettid(), __func__); - return 0; - } - - virtual uint64_t get_file_size() override { - dbg_default_debug("----GET FILE SIZE key is [{}].", this->key); - check_update(); - return this->file_bytes.get()->size; - } - - virtual ~ObjectPoolKeyINode() { - dbg_default_info("[{}] entering {}.", gettid(), __func__); - file_bytes.reset(nullptr); - dbg_default_info("[{}] leaving {}.", gettid(), __func__); - } - private: virtual void update_contents() override { dbg_default_debug("\n \n ----OBJP keyInode key is:[{}] - update content [{}] entering {}.", this->key, gettid(), __func__); @@ -1000,9 +963,11 @@ class ObjectPoolKeyINode : public FuseDataINode { this->previous_version = access_ptr->previous_version; this->previous_version_by_key = access_ptr->previous_version_by_key; blob = access_ptr->blob; - this->file_bytes.get()->size = blob.size; - this->file_bytes.get()->bytes = static_cast(malloc(blob.size)); - memcpy(this->file_bytes.get()->bytes, blob.bytes, blob.size); + // this->file_bytes.get()->size = blob.size; + // this->file_bytes.get()->bytes = static_cast(malloc(blob.size)); + // memcpy(this->file_bytes.get()->bytes, blob.bytes, blob.size); + this->cached_data.bytes.resize(blob.size); + memcpy(this->cached_data.bytes.data(), blob.bytes, blob.size); return; } dbg_default_trace("\n \n ----OBJP keyInode update content [{}] leaving {}.", gettid(), __func__); @@ -1250,7 +1215,7 @@ class FuseClientContext { stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; break; case INodeType::KEY: - stbuf.st_mode = S_IFREG | 0444; + stbuf.st_mode = S_IFREG | 0744; // TODO set 0444 for unimplemented stbuf.st_size = pfci->get_file_size(); stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; diff --git a/src/service/fuse/fuse_client_hl.cpp b/src/service/fuse/fuse_client_hl.cpp index dcce9bce..3234b9a5 100644 --- a/src/service/fuse/fuse_client_hl.cpp +++ b/src/service/fuse/fuse_client_hl.cpp @@ -109,7 +109,7 @@ static int cascade_fs_open(const char* path, struct fuse_file_info* fi) { // TODO check O_ACCMODE, also check if dir ?? auto node = fcc()->get(path); if(fi->flags & O_CREAT && node == nullptr) { - node = fcc()->tree->add_op_key(path); + node = fcc()->add_op_key(path); if(node == nullptr) { return -ENOTSUP; } @@ -117,7 +117,7 @@ static int cascade_fs_open(const char* path, struct fuse_file_info* fi) { if(node == nullptr) { return -ENOENT; } - if(node->data.flag & OP_DIR) { + if(node->data.flag & DIR_FLAG) { return -ENOTSUP; } if(fi->flags & O_TRUNC) { @@ -141,7 +141,7 @@ static int cascade_fs_read(const char* path, char* buf, size_t size, off_t offse if(node == nullptr) { return -ENOENT; } - if(node->data.flag & OP_DIR) { // TODO diff error + if(node->data.flag & DIR_FLAG) { // TODO diff error return -EACCES; } auto& bytes = node->data.bytes; @@ -167,7 +167,7 @@ static int cascade_fs_write(const char* path, const char* buf, size_t size, if(node == nullptr) { return -ENOENT; } - if(node->data.flag & OP_DIR) { // TODO diff error + if(node->data.flag & DIR_FLAG) { // TODO diff error return -ENOTSUP; } auto& bytes = node->data.bytes; @@ -195,24 +195,30 @@ static int cascade_fs_release(const char* path, struct fuse_file_info* fi) { if(node == nullptr) { return -ENOENT; } - if(node->data.flag & OP_DIR) { + if(node->data.flag & DIR_FLAG) { return -ENOTSUP; } - return fcc()->put(node); - return 0; + return fcc()->put_to_capi(node); } static int cascade_fs_mkdir(const char* path, mode_t mode) { if(fcc()->get(path)) { return -EEXIST; } - auto op_root = fcc()->tree->object_pool_root(path); + auto op_root = fcc()->nearest_object_pool_root(path); if(op_root == nullptr) { + persistent::version_t ver = fcc()->is_snapshot(path); + if(ver != CURRENT_VERSION && ver <= fcc()->max_ver) { + dbg_info(DL, "{}", ver); + + fcc()->add_snapshot(ver); + return 0; + } return -EACCES; } - auto res = fcc()->tree->add_op_key_dir(path); + auto res = fcc()->add_op_key_dir(path); if(res) { - fcc()->local_dirs.insert(path); + fcc()->local_latest_dirs.insert(path); // TODO err if nullptr } @@ -225,7 +231,7 @@ static int cascade_fs_unlink(const char* path) { if(node == nullptr) { return -ENOENT; } - if(node->data.flag & OP_DIR) { + if(node->data.flag & DIR_FLAG) { return -EISDIR; } // TODO check open @@ -245,10 +251,10 @@ static int cascade_fs_rmdir(const char* path) { if(node == nullptr) { return -ENOENT; } - if(!(node->data.flag & OP_DIR)) { + if(!(node->data.flag & DIR_FLAG)) { return -ENOTDIR; } - if(node->data.flag & (OP_PREFIX_DIR | OP_ROOT_DIR)) { + if((node->data.flag & KEY_DIR) == 0) { return -EACCES; } if(!node->children.empty()) { @@ -256,7 +262,7 @@ static int cascade_fs_rmdir(const char* path) { } // remove - fcc()->local_dirs.erase(path); + fcc()->local_latest_dirs.erase(path); node->parent->children.erase(node->label); delete node; @@ -269,11 +275,11 @@ static int cascade_fs_truncate(const char* path, off_t size, if(node == nullptr) { return -ENOENT; } - if(node->data.flag & OP_DIR) { + if(node->data.flag & DIR_FLAG) { return -EINVAL; } node->data.bytes.resize(size, 0); - return fcc()->put(node); + return fcc()->put_to_capi(node); } int set_buffer(char* dest, size_t size, const char* src, size_t len = 0) { @@ -304,6 +310,8 @@ static int cascade_fs_utimens(const char*, const struct timespec tv[2], struct f } #ifdef HAVE_SETXATTR + +/* static int cascade_fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags) { if(flags & XATTR_CREATE) { @@ -311,7 +319,7 @@ static int cascade_fs_setxattr(const char* path, const char* name, const char* v } dbg_info(DL, "{}, {}, {}", path, name, value); // TODO save into node data - if(strcmp(path, ROOT) == 0) { + if(strcmp(path, fcc()->ROOT) == 0) { if(!fcc()->latest && strcmp(name, "user.cascade.version") == 0) { fcc()->ver = strtoll(value, nullptr, 0); // TODO @@ -325,17 +333,13 @@ static int cascade_fs_setxattr(const char* path, const char* name, const char* v } return -EPERM; } +*/ static int cascade_fs_getxattr(const char* path, const char* name, char* value, size_t size) { // TODO need to first apt get-install attr - if(strcmp(path, ROOT) == 0) { - if(strcmp(name, "user.cascade.version") == 0) { - std::string v = std::to_string(fcc()->ver); - return set_buffer(value, size, v.c_str()); - } else if(strcmp(name, "user.cascade.latest") == 0) { - return set_buffer(value, size, fcc()->latest ? "1" : "0"); - } else if(strcmp(name, "user.cascade.largest_known_version") == 0) { + if(strcmp(path, fcc()->ROOT.c_str()) == 0) { + if(strcmp(name, "user.cascade.largest_known_version") == 0) { std::string v = std::to_string(fcc()->max_ver); return set_buffer(value, size, v.c_str()); } @@ -345,14 +349,14 @@ static int cascade_fs_getxattr(const char* path, const char* name, char* value, static int cascade_fs_listxattr(const char* path, char* list, size_t size) { // TODO lesson learned :(. returned 0 instead of length. check over all return types - if(strcmp(path, ROOT) == 0) { + if(strcmp(path, fcc()->ROOT.c_str()) == 0) { // ^ 1hr+ bug const char names[] = "user.cascade.largest_known_version" - "\0" - "user.cascade.version" - "\0" - "user.cascade.latest" "\0"; + // "user.cascade.version" + // "\0" + // "user.cascade.latest" + // "\0"; // very weird behavior with \0 termination. strings and determining sizes did not work well // ^ 2hr+ bug return set_buffer(list, size, names, sizeof(names) / sizeof(names[0]) - 1); @@ -360,6 +364,7 @@ static int cascade_fs_listxattr(const char* path, char* list, size_t size) { const char empty[] = ""; return set_buffer(list, size, empty, 0); } + #endif // TODO create_object_pool, remove_object_pool, get_object_pool, op_remove @@ -382,7 +387,7 @@ static const struct fuse_operations cascade_fs_oper = { // .flush = cascade_fs_flush, .release = cascade_fs_release, #ifdef HAVE_SETXATTR - .setxattr = cascade_fs_setxattr, + // .setxattr = cascade_fs_setxattr, .getxattr = cascade_fs_getxattr, .listxattr = cascade_fs_listxattr, #endif diff --git a/src/service/fuse/path_tree.hpp b/src/service/fuse/path_tree.hpp index 501b363b..fc9c4592 100644 --- a/src/service/fuse/path_tree.hpp +++ b/src/service/fuse/path_tree.hpp @@ -131,6 +131,7 @@ struct PathTree { } PathTree* extract(const fs::path& path) { + // TODO use?? PathTree* cur = get(path); if(cur == nullptr || cur->parent == nullptr) { return nullptr; From 4df5f68fe2ca238a1f79b5d987e0b81a8a14d7f3 Mon Sep 17 00:00:00 2001 From: Michael Ye Date: Fri, 31 Mar 2023 12:40:58 -0400 Subject: [PATCH 10/10] timestamp snapshotting --- scripts/util/cascade_runner.sh | 10 +- src/service/fuse/fcc_hl.hpp | 186 +++++++++++++++++++++------- src/service/fuse/fuse_client_hl.cpp | 35 +++--- 3 files changed, 162 insertions(+), 69 deletions(-) diff --git a/scripts/util/cascade_runner.sh b/scripts/util/cascade_runner.sh index 220f9e41..ecdcbde3 100755 --- a/scripts/util/cascade_runner.sh +++ b/scripts/util/cascade_runner.sh @@ -6,6 +6,7 @@ SCRIPT_NAME=$(basename "${BASH_SOURCE[0]}") HELP_STRING="A helper script to set up a local cascade instance Usage: $SCRIPT_NAME [options] -h Print this message and exit. + -a ARGS Args passed to resulting command. -d D CFG directory location (defaults to cfg). -n N Sets a range from 0..N-1 for range commands. @@ -44,6 +45,8 @@ CLIENT_COMMAND="cascade_client" SERVER_COMMAND="cascade_server" LOG_FILE="node_info.log" +ARGS="" + if ! command -v $SERVER_COMMAND &>/dev/null; then echo "$SERVER_COMMAND could not be found" exit_abnormal @@ -81,9 +84,10 @@ foreach() { } main() { - while getopts 'hd:n:f:F:krsc:u:v:' opt; do + while getopts 'ha:d:n:f:F:krsc:u:v:' opt; do case "$opt" in h) HELP=true ;; + a) ARGS="$OPTARG" ;; d) CFG_DIR="$OPTARG" ;; n) NODES="$OPTARG" @@ -123,9 +127,9 @@ main() { mkdir -p "$MOUNT" if [[ "$FUSE_FOREGROUND" = true ]]; then - "$FUSE_COMMAND" -s -f "$MOUNT" + "$FUSE_COMMAND" -s -f "$MOUNT" $ARGS else - "$FUSE_COMMAND" -s -f "$MOUNT" &>"$LOG_FILE" & + "$FUSE_COMMAND" -s -f "$MOUNT" $ARGS &>"$LOG_FILE" & fi cd - &>/dev/null diff --git a/src/service/fuse/fcc_hl.hpp b/src/service/fuse/fcc_hl.hpp index 2ccfdcde..44d99308 100644 --- a/src/service/fuse/fcc_hl.hpp +++ b/src/service/fuse/fcc_hl.hpp @@ -34,6 +34,13 @@ enum NodeFlag : uint32_t { SNAPSHOT_TIME_DIR = 1 << 9, }; +/* +op_get_by_time +op_get_size <-- create new function for content size ??? +VT template type ?? +op_list_keys_by_time +*/ + const uint32_t FILE_FLAG = KEY_FILE | METADATA_INFO_FILE; const uint32_t DIR_FLAG = ~FILE_FLAG; const uint32_t OP_FLAG = OP_PREFIX_DIR | OP_ROOT_DIR | KEY_DIR | KEY_FILE; @@ -44,7 +51,9 @@ struct NodeData { uint64_t timestamp; std::vector bytes; - NodeData(const NodeFlag flag) : flag(flag), timestamp(0) { + bool writeable; + + NodeData(const NodeFlag flag) : flag(flag), timestamp(0), writeable(false) { } }; @@ -63,21 +72,27 @@ struct FuseClientContext { const fs::path ROOT = "/"; ServiceClientAPI& capi; + bool version_snapshot; persistent::version_t max_ver; + uint64_t max_timestamp; // timestamp in microsecond std::set local_latest_dirs; std::set snapshots; + std::set snapshots_by_time; time_t update_interval; time_t last_update_sec; - FuseClientContext() : capi(ServiceClientAPI::get_service_client()) { + FuseClientContext(int update_int, bool ver_snap) : capi(ServiceClientAPI::get_service_client()) { DL = LoggerFactory::createLogger("fuse_client", spdlog::level::from_str(derecho::getConfString(CONF_LOGGER_DEFAULT_LOG_LEVEL))); DL->set_pattern("[%T][%n][%^%l%$] %v"); max_ver = 0; + max_timestamp = 0; - update_interval = 15; + version_snapshot = ver_snap; + dbg_info(DL, "snapshot type: {}", version_snapshot ? "version" : "timestamp"); + update_interval = update_int; last_update_sec = 0; root = std::make_unique(ROOT, NodeData(ROOT_DIR)); @@ -145,16 +160,44 @@ struct FuseClientContext { return object_pool_root(node); } - persistent::version_t is_snapshot(const fs::path& path) { + bool add_snapshot(const fs::path& path) { if(path.parent_path() != SNAPSHOT_PATH) { - return CURRENT_VERSION; + return false; } try { - return std::stoull(path.filename()); + unsigned long long ts_us = std::stoull(path.filename()); + persistent::version_t ver = ts_us; + if(version_snapshot && ver <= max_ver) { + add_snapshot_folder(ver); + return true; + } + if(!version_snapshot && ts_us <= max_timestamp) { + add_snapshot_folder_by_time(ts_us); + return true; + } } catch(std::invalid_argument const& ex) { - return CURRENT_VERSION; } catch(std::out_of_range const& ex) { - return CURRENT_VERSION; + } + return false; + } + + void add_snapshot_folder(persistent::version_t ver) { + auto snapshot = SNAPSHOT_PATH; + snapshot += "/" + std::to_string(ver); + auto res = add_snapshot_time(snapshot); + dbg_info(DL, "adding {}", snapshot); + if(res != nullptr) { + fill_at(snapshot, ver); + } + } + + void add_snapshot_folder_by_time(uint64_t ts_us) { + auto snapshot = SNAPSHOT_PATH; + snapshot += "/" + std::to_string(ts_us); + auto res = add_snapshot_time(snapshot); + dbg_info(DL, "adding {}", snapshot); + if(res != nullptr) { + fill_at_by_time(snapshot, ts_us); } } @@ -200,14 +243,37 @@ struct FuseClientContext { return 0; } + void fill_op_meta(const fs::path& prefix, const std::string& op_root) { + auto opm = capi.find_object_pool(op_root); + json j{{"valid", opm.is_valid()}, + {"null", opm.is_null()}}; + if(opm.is_valid() && !opm.is_null()) { + j.emplace("pathname", opm.pathname); + j.emplace("version", opm.version); + j.emplace("timestamp_us", opm.timestamp_us); + j.emplace("previous_version", opm.previous_version); + j.emplace("previous_version_by_key", opm.previous_version_by_key); + j.emplace("subgroup_type", std::to_string(opm.subgroup_type_index) + "-->" + DefaultObjectPoolMetadataType::subgroup_type_order[opm.subgroup_type_index].name()); + j.emplace("subgroup_index", opm.subgroup_index); + j.emplace("sharding_policy", std::to_string(opm.sharding_policy)); + j.emplace("deleted", opm.deleted); + } + auto op_root_meta_path = prefix; + op_root_meta_path += METADATA_PATH; + op_root_meta_path += op_root; + add_op_info(op_root_meta_path, j.dump(2)); + } + void fill_at(const fs::path& prefix, persistent::version_t ver) { // TODO no version for list object pools auto str_paths = capi.list_object_pools(false, true); - auto meta_path = prefix; - meta_path += METADATA_PATH; - root->set(meta_path, - NodeData(METADATA_PREFIX_DIR), NodeData(METADATA_PREFIX_DIR)); + if(ver == CURRENT_VERSION) { + auto meta_path = prefix; + meta_path += METADATA_PATH; + root->set(meta_path, + NodeData(METADATA_PREFIX_DIR), NodeData(METADATA_PREFIX_DIR)); + } for(const std::string& op_root : str_paths) { auto op_root_path = prefix; op_root_path += op_root; @@ -216,24 +282,9 @@ struct FuseClientContext { continue; } - auto opm = capi.find_object_pool(op_root); - json j{{"valid", opm.is_valid()}, - {"null", opm.is_null()}}; - if(opm.is_valid() && !opm.is_null()) { - j.emplace("pathname", opm.pathname); - j.emplace("version", opm.version); - j.emplace("timestamp_us", opm.timestamp_us); - j.emplace("previous_version", opm.previous_version); - j.emplace("previous_version_by_key", opm.previous_version_by_key); - j.emplace("subgroup_type", std::to_string(opm.subgroup_type_index) + "-->" + DefaultObjectPoolMetadataType::subgroup_type_order[opm.subgroup_type_index].name()); - j.emplace("subgroup_index", opm.subgroup_index); - j.emplace("sharding_policy", std::to_string(opm.sharding_policy)); - j.emplace("deleted", opm.deleted); + if(ver == CURRENT_VERSION) { + fill_op_meta(prefix, op_root); } - auto op_root_meta_path = prefix; - op_root_meta_path += METADATA_PATH; - op_root_meta_path += op_root; - add_op_info(op_root_meta_path, j.dump(2)); auto keys = get_keys(op_root, ver); std::sort(keys.begin(), keys.end(), std::greater<>()); @@ -242,9 +293,8 @@ struct FuseClientContext { auto key_path = prefix; key_path += k; auto node = add_op_key(key_path); - if(node == nullptr) { - dbg_info(DL, "did not add {}", key_path); - } else { + // colliding keys do not get added + if(node != nullptr) { get_contents(node, k, ver); // dbg_info(DL, "file: {}", std::quoted(reinterpret_cast(node->data.bytes.data()))); } @@ -252,15 +302,31 @@ struct FuseClientContext { } } - void add_snapshot(persistent::version_t ver) { - auto snapshot = SNAPSHOT_PATH; - snapshot += "/" + std::to_string(ver); - auto res = add_snapshot_time(snapshot); - dbg_info(DL, "adding {}", snapshot); - if(res != nullptr) { - dbg_info(DL, "ok"); - fill_at(snapshot, ver); - dbg_info(DL, "filled"); + void fill_at_by_time(const fs::path& prefix, uint64_t ts_us) { + // TODO no version for list object pools + auto str_paths = capi.list_object_pools(false, true); + + for(const std::string& op_root : str_paths) { + auto op_root_path = prefix; + op_root_path += op_root; + auto op_root_node = add_op_root(op_root_path); + if(op_root_node == nullptr) { + continue; + } + + auto keys = get_keys_by_time(op_root, ts_us); + std::sort(keys.begin(), keys.end(), std::greater<>()); + // sort removes files colliding with directory + for(const auto& k : keys) { + auto key_path = prefix; + key_path += k; + // colliding keys do not get added + auto node = add_op_key(key_path); + if(node != nullptr) { + get_contents_by_time(node, k, ts_us); + // dbg_info(DL, "file: {}", std::quoted(reinterpret_cast(node->data.bytes.data()))); + } + } } } @@ -294,14 +360,19 @@ struct FuseClientContext { stbuf->st_nlink = 1; stbuf->st_uid = fuse_get_context()->uid; stbuf->st_gid = fuse_get_context()->gid; - stbuf->st_atim = timespec{last_update_sec, 0}; // TODO merge timestamp for dirs, update during set int64_t sec = node->data.timestamp / 1'000'000; int64_t nano = (node->data.timestamp % 1'000'000) * 1000; stbuf->st_mtim = timespec{sec, nano}; stbuf->st_ctim = stbuf->st_mtim; + stbuf->st_atim = timespec{last_update_sec, 0}; + if(uint64_t(last_update_sec) * 1'000'000 < node->data.timestamp) { + stbuf->st_atim = stbuf->st_mtim; + } // - at prefix dir location add .info file ??? + // TODO timestamps messing with vim?? + if(node->data.flag & DIR_FLAG) { if(node->data.flag & OP_PREFIX_DIR) { stbuf->st_mode = S_IFDIR | 0555; @@ -342,7 +413,6 @@ struct FuseClientContext { } void get_contents(Node* node, const std::string& path, persistent::version_t ver) { - // persistent::version_t version = CURRENT_VERSION; auto result = capi.get(path, ver, true); // TODO only get file contents on open @@ -350,7 +420,11 @@ struct FuseClientContext { auto reply = reply_future.second.get(); if(ver == CURRENT_VERSION) { max_ver = std::max(max_ver, reply.version); + max_timestamp = std::max(max_timestamp, reply.timestamp_us); + + node->data.writeable = true; } + // TODO std::move ?? Blob blob = reply.blob; std::vector bytes(blob.bytes, blob.bytes + blob.size); node->data.bytes = bytes; @@ -359,12 +433,30 @@ struct FuseClientContext { } } + // not to be called by latest + void get_contents_by_time(Node* node, const std::string& path, uint64_t ts_us) { + auto result = capi.get_by_time(path, ts_us, true); + // TODO only get file contents on open + + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + Blob blob = reply.blob; + // TODO std::move ?? + std::vector bytes(blob.bytes, blob.bytes + blob.size); + node->data.bytes = bytes; + node->data.timestamp = reply.timestamp_us; + return; + } + } + std::vector get_keys(const std::string& path, persistent::version_t ver) { - // persistent::version_t version = CURRENT_VERSION; auto future_result = capi.list_keys(ver, true, path); - auto reply = capi.wait_list_keys(future_result); - // TODO remove keys that collide with directory (key is a prefix of another key) - return reply; + return capi.wait_list_keys(future_result); + } + + std::vector get_keys_by_time(const std::string& path, uint64_t ts_us) { + auto future_result = capi.list_keys_by_time(ts_us, true, path); + return capi.wait_list_keys(future_result); } // make a trash folder? (move on delete) diff --git a/src/service/fuse/fuse_client_hl.cpp b/src/service/fuse/fuse_client_hl.cpp index 3234b9a5..8fc95341 100644 --- a/src/service/fuse/fuse_client_hl.cpp +++ b/src/service/fuse/fuse_client_hl.cpp @@ -20,16 +20,19 @@ using namespace derecho::cascade; +// TODO segfault debugging +// stat on invalid path + struct cli_options { const char* client_dir; int update_interval; - int flag; + int by_version; }; static cli_options options = { .client_dir = nullptr, - .update_interval = 1, - .flag = 0, + .update_interval = 15, + .by_version = 0, }; #define OPTION(t, p) \ @@ -38,14 +41,14 @@ static cli_options options = { static const struct fuse_opt option_spec[] = { OPTION("--client=%s", client_dir), OPTION("--update-interval=%d", update_interval), - OPTION("--flag", flag), + OPTION("--by_version", by_version), FUSE_OPT_END}; static void show_help(const char* progname) { printf("usage: %s [options] \n\n", progname); - printf(" --update-interval= Update-rate of file system contents\n" + printf(" --update-interval= Update-rate of file system contents (default: 15)\n" " --client= Client directory\n" - " --flag Flag that does nothing\n" + " --by_version Snapshot by version number rather than timestamp in microseconds\n" "\n"); } @@ -59,7 +62,8 @@ static void* cascade_fs_init(struct fuse_conn_info* conn, struct fuse_config* cfg) { // TODO why read conf_layout_json_layout? // TODO don't like no control over derecho config - return new FuseClientContext(); + + return new FuseClientContext(options.update_interval, options.by_version); } static void cascade_fs_destroy(void* private_data) { @@ -167,7 +171,7 @@ static int cascade_fs_write(const char* path, const char* buf, size_t size, if(node == nullptr) { return -ENOENT; } - if(node->data.flag & DIR_FLAG) { // TODO diff error + if(node->data.flag & DIR_FLAG || !node->data.writeable) { // TODO diff error return -ENOTSUP; } auto& bytes = node->data.bytes; @@ -195,7 +199,7 @@ static int cascade_fs_release(const char* path, struct fuse_file_info* fi) { if(node == nullptr) { return -ENOENT; } - if(node->data.flag & DIR_FLAG) { + if(node->data.flag & DIR_FLAG || !node->data.writeable) { return -ENOTSUP; } return fcc()->put_to_capi(node); @@ -207,14 +211,7 @@ static int cascade_fs_mkdir(const char* path, mode_t mode) { } auto op_root = fcc()->nearest_object_pool_root(path); if(op_root == nullptr) { - persistent::version_t ver = fcc()->is_snapshot(path); - if(ver != CURRENT_VERSION && ver <= fcc()->max_ver) { - dbg_info(DL, "{}", ver); - - fcc()->add_snapshot(ver); - return 0; - } - return -EACCES; + return fcc()->add_snapshot(path) ? 0 : -EACCES; } auto res = fcc()->add_op_key_dir(path); if(res) { @@ -275,7 +272,7 @@ static int cascade_fs_truncate(const char* path, off_t size, if(node == nullptr) { return -ENOENT; } - if(node->data.flag & DIR_FLAG) { + if(node->data.flag & DIR_FLAG || !node->data.writeable) { return -EINVAL; } node->data.bytes.resize(size, 0); @@ -317,7 +314,7 @@ static int cascade_fs_setxattr(const char* path, const char* name, const char* v if(flags & XATTR_CREATE) { return -EPERM; } - dbg_info(DL, "{}, {}, {}", path, name, value); + dbg_default_info("{}, {}, {}", path, name, value); // TODO save into node data if(strcmp(path, fcc()->ROOT) == 0) { if(!fcc()->latest && strcmp(name, "user.cascade.version") == 0) {