diff --git a/.clang-format b/.clang-format index 7e4a8374..c7e91cf5 100644 --- a/.clang-format +++ b/.clang-format @@ -12,7 +12,7 @@ AllowShortLoopsOnASingleLine: true AllowShortFunctionsOnASingleLine: All AlwaysBreakTemplateDeclarations: true AlwaysBreakBeforeMultilineStrings: false -BreakBeforeBinaryOperators: true +BreakBeforeBinaryOperators: true BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: false BinPackParameters: true @@ -53,4 +53,3 @@ ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] SpaceBeforeParens: Never DisableFormat: false ... - diff --git a/include/cascade/object.hpp b/include/cascade/object.hpp index 997c0357..a7b49e3f 100644 --- a/include/cascade/object.hpp +++ b/include/cascade/object.hpp @@ -1,14 +1,15 @@ #pragma once #include +#include #include #include #include +#include #include #include #include -#include -#include #include +#include #include #include @@ -20,8 +21,8 @@ using namespace persistent; using namespace std::chrono_literals; -namespace derecho{ -namespace cascade{ +namespace derecho { +namespace cascade { enum object_memory_mode_t { DEFAULT, @@ -29,7 +30,7 @@ enum object_memory_mode_t { BLOB_GENERATOR, }; -using blob_generator_func_t = std::function; +using blob_generator_func_t = std::function; class Blob : public mutils::ByteRepresentable { public: @@ -40,8 +41,7 @@ class Blob : public mutils::ByteRepresentable { // for BLOB_GENERATOR mode only blob_generator_func_t blob_generator; - object_memory_mode_t memory_mode; - + object_memory_mode_t memory_mode; // constructor - copy to own the data Blob(const uint8_t* const b, const decltype(size) s); @@ -81,53 +81,54 @@ class Blob : public mutils::ByteRepresentable { static std::unique_ptr from_bytes(mutils::DeserializationManager*, const uint8_t* const v); static mutils::context_ptr from_bytes_noalloc( - mutils::DeserializationManager* ctx, - const uint8_t* const v); + mutils::DeserializationManager* ctx, + const uint8_t* const v); static mutils::context_ptr from_bytes_noalloc_const( - mutils::DeserializationManager* ctx, - const uint8_t* const v); + mutils::DeserializationManager* ctx, + const uint8_t* const v); }; #define INVALID_UINT64_OBJECT_KEY (0xffffffffffffffffLLU) class ObjectWithUInt64Key : public mutils::ByteRepresentable, - public ICascadeObject, + public ICascadeObject, public IKeepTimestamp, public IVerifyPreviousVersion #ifdef ENABLE_EVALUATION - , public IHasMessageID + , + public IHasMessageID #endif - { +{ public: #ifdef ENABLE_EVALUATION - mutable uint64_t message_id; + mutable uint64_t message_id; #endif - mutable persistent::version_t version; - mutable uint64_t timestamp_us; - mutable persistent::version_t previous_version; // previous version, INVALID_VERSION for the first version - mutable persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. - uint64_t key; // object_id - Blob blob; // the object + mutable persistent::version_t version; + mutable uint64_t timestamp_us; + mutable persistent::version_t previous_version; // previous version, INVALID_VERSION for the first version + mutable persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. + uint64_t key; // object_id + Blob blob; // the object // bool operator==(const ObjectWithUInt64Key& other); // constructor 0 : copy constructor - ObjectWithUInt64Key(const uint64_t _key, + ObjectWithUInt64Key(const uint64_t _key, const Blob& _blob); // constructor 0.5 : copy/emplace constructor ObjectWithUInt64Key( #ifdef ENABLE_EVALUATION - const uint64_t _message_id, + const uint64_t _message_id, #endif - const persistent::version_t _version, - const uint64_t _timestamp_us, - const persistent::version_t _previous_version, - const persistent::version_t _previous_version_by_key, - const uint64_t _key, - const Blob& _blob, - bool is_emplaced = false); + const persistent::version_t _version, + const uint64_t _timestamp_us, + const persistent::version_t _previous_version, + const persistent::version_t _previous_version_by_key, + const uint64_t _key, + const Blob& _blob, + bool is_emplaced = false); // constructor 1 : copy constructor ObjectWithUInt64Key(const uint64_t _key, @@ -137,15 +138,15 @@ class ObjectWithUInt64Key : public mutils::ByteRepresentable, // constructor 1.5 : copy constructor ObjectWithUInt64Key( #ifdef ENABLE_EVALUATION - const uint64_t _message_id, + const uint64_t _message_id, #endif - const persistent::version_t _version, - const uint64_t _timestamp_us, - const persistent::version_t _previous_version, - const persistent::version_t _previous_version_by_key, - const uint64_t _key, - const uint8_t* const _b, - const std::size_t _s); + const persistent::version_t _version, + const uint64_t _timestamp_us, + const persistent::version_t _previous_version, + const persistent::version_t _previous_version_by_key, + const uint64_t _key, + const uint8_t* const _b, + const std::size_t _s); // TODO: we need a move version for the deserializer. @@ -165,15 +166,15 @@ class ObjectWithUInt64Key : public mutils::ByteRepresentable, // constructor 5.5 : using delayed instratiator with message generator ObjectWithUInt64Key( #ifdef ENABLE_EVALUATION - const uint64_t _message_id, + const uint64_t _message_id, #endif - const persistent::version_t _version, - const uint64_t _timestamp_us, - const persistent::version_t _previous_version, - const persistent::version_t _previous_version_by_key, - const uint64_t _key, - const blob_generator_func_t& _message_generator, - const std::size_t _s); + const persistent::version_t _version, + const uint64_t _timestamp_us, + const persistent::version_t _previous_version, + const persistent::version_t _previous_version_by_key, + const uint64_t _key, + const blob_generator_func_t& _message_generator, + const std::size_t _s); virtual const uint64_t& get_key_ref() const override; virtual bool is_null() const override; @@ -198,11 +199,11 @@ class ObjectWithUInt64Key : public mutils::ByteRepresentable, void ensure_registerd(mutils::DeserializationManager&) {} static std::unique_ptr from_bytes(mutils::DeserializationManager*, const uint8_t* const v); static mutils::context_ptr from_bytes_noalloc( - mutils::DeserializationManager* ctx, - const uint8_t* const v); + mutils::DeserializationManager* ctx, + const uint8_t* const v); static mutils::context_ptr from_bytes_noalloc_const( - mutils::DeserializationManager* ctx, - const uint8_t* const v); + mutils::DeserializationManager* ctx, + const uint8_t* const v); // IK and IV for volatile cascade store static uint64_t IK; @@ -210,67 +211,70 @@ class ObjectWithUInt64Key : public mutils::ByteRepresentable, }; inline std::ostream& operator<<(std::ostream& out, const Blob& b) { - out << "[size:" << b.size << ", data:" << std::hex; - if(b.size > 0) { - uint32_t i = 0; - for(i = 0; i < 8 && i < b.size; i++) { - out << " " << b.bytes[i]; - } - if(i < b.size) { - out << "..."; - } - } - out << std::dec << "]"; + out << "[size:" << b.size; + // out << ", data:"; + // out << std::quoted(reinterpret_cast(b.bytes)); + // if(b.size > 0) { + // uint32_t i = 0; + // for(i = 0; i < 8 && i < b.size; i++) { + // out << " " << b.bytes[i]; + // } + // if(i < b.size) { + // out << "..."; + // } + // } + out << "]"; return out; } inline std::ostream& operator<<(std::ostream& out, const ObjectWithUInt64Key& o) { - out << "ObjectWithUInt64Key{ver: 0x" << std::hex << o.version << std::dec - << ", ts(us): " << o.timestamp_us + out << "ObjectWithUInt64Key{ver: 0x" << std::hex << o.version << std::dec + << ", ts(us): " << o.timestamp_us << ", prev_ver: " << std::hex << o.previous_version << std::dec << ", prev_ver_by_key: " << std::hex << o.previous_version_by_key << std::dec - << ", id:" << o.key + << ", id:" << o.key << ", data:" << o.blob << "}"; return out; } class ObjectWithStringKey : public mutils::ByteRepresentable, - public ICascadeObject, + public ICascadeObject, public IKeepTimestamp, public IVerifyPreviousVersion #ifdef ENABLE_EVALUATION - ,public IHasMessageID + , + public IHasMessageID #endif - { +{ public: #ifdef ENABLE_EVALUATION - mutable uint64_t message_id; + mutable uint64_t message_id; #endif - mutable persistent::version_t version; // object version - mutable uint64_t timestamp_us; // timestamp in microsecond - mutable persistent::version_t previous_version; // previous version, INVALID_VERSION for the first version. - mutable persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. - std::string key; // object_id - Blob blob; // the object data + mutable persistent::version_t version; // object version + mutable uint64_t timestamp_us; // timestamp in microsecond + mutable persistent::version_t previous_version; // previous version, INVALID_VERSION for the first version. + mutable persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. + std::string key; // object_id + Blob blob; // the object data // bool operator==(const ObjectWithStringKey& other); // constructor 0 : copy constructor - ObjectWithStringKey(const std::string& _key, + ObjectWithStringKey(const std::string& _key, const Blob& _blob); // constructor 0.5 : copy/in-place constructor ObjectWithStringKey( #ifdef ENABLE_EVALUATION - const uint64_t message_id, + const uint64_t message_id, #endif - const persistent::version_t _version, - const uint64_t _timestamp_us, - const persistent::version_t _previous_version, - const persistent::version_t _previous_version_by_key, - const std::string& _key, - const Blob& _blob, - bool is_emplaced = false); + const persistent::version_t _version, + const uint64_t _timestamp_us, + const persistent::version_t _previous_version, + const persistent::version_t _previous_version_by_key, + const std::string& _key, + const Blob& _blob, + bool is_emplaced = false); // constructor 1 : copy consotructor ObjectWithStringKey(const std::string& _key, @@ -280,15 +284,15 @@ class ObjectWithStringKey : public mutils::ByteRepresentable, // constructor 1.5 : copy constructor ObjectWithStringKey( #ifdef ENABLE_EVALUATION - const uint64_t message_id, + const uint64_t message_id, #endif - const persistent::version_t _version, - const uint64_t _timestamp_us, - const persistent::version_t _previous_version, - const persistent::version_t _previous_version_by_key, - const std::string& _key, - const uint8_t* const _b, - const std::size_t _s); + const persistent::version_t _version, + const uint64_t _timestamp_us, + const persistent::version_t _previous_version, + const persistent::version_t _previous_version_by_key, + const std::string& _key, + const uint8_t* const _b, + const std::size_t _s); // TODO: we need a move version for the deserializer. @@ -308,15 +312,15 @@ class ObjectWithStringKey : public mutils::ByteRepresentable, // constructor 5.5 : using delayed instatiator withe message generator ObjectWithStringKey( #ifdef ENABLE_EVALUATION - const uint64_t message_id, + const uint64_t message_id, #endif - const persistent::version_t _version, - const uint64_t _timestamp_us, - const persistent::version_t _previous_version, - const persistent::version_t _previous_version_by_key, - const std::string& _key, - const blob_generator_func_t& _message_generator, - const std::size_t _s); + const persistent::version_t _version, + const uint64_t _timestamp_us, + const persistent::version_t _previous_version, + const persistent::version_t _previous_version_by_key, + const std::string& _key, + const blob_generator_func_t& _message_generator, + const std::size_t _s); virtual const std::string& get_key_ref() const override; virtual bool is_null() const override; @@ -333,18 +337,18 @@ class ObjectWithStringKey : public mutils::ByteRepresentable, virtual uint64_t get_message_id() const override; #endif -// DEFAULT_SERIALIZATION_SUPPORT(ObjectWithStringKey, version, timestamp_us, previous_version, previous_version_by_key, key, blob); + // DEFAULT_SERIALIZATION_SUPPORT(ObjectWithStringKey, version, timestamp_us, previous_version, previous_version_by_key, key, blob); std::size_t to_bytes(uint8_t* v) const; std::size_t bytes_size() const; void post_object(const std::function& f) const; void ensure_registerd(mutils::DeserializationManager&) {} static std::unique_ptr from_bytes(mutils::DeserializationManager*, const uint8_t* const v); static mutils::context_ptr from_bytes_noalloc( - mutils::DeserializationManager* ctx, - const uint8_t* const v); + mutils::DeserializationManager* ctx, + const uint8_t* const v); static mutils::context_ptr from_bytes_noalloc_const( - mutils::DeserializationManager* ctx, - const uint8_t* const v); + mutils::DeserializationManager* ctx, + const uint8_t* const v); // IK and IV for volatile cascade store static std::string IK; @@ -352,15 +356,15 @@ class ObjectWithStringKey : public mutils::ByteRepresentable, }; inline std::ostream& operator<<(std::ostream& out, const ObjectWithStringKey& o) { - out << "ObjectWithStringKey{" + out << "ObjectWithStringKey{" #ifdef ENABLE_EVALUATION << "msg_id: " << o.message_id #endif - << "ver: 0x" << std::hex << o.version << std::dec + << "ver: 0x" << std::hex << o.version << std::dec << ", ts: " << o.timestamp_us << ", prev_ver: " << std::hex << o.previous_version << std::dec << ", prev_ver_by_key: " << std::hex << o.previous_version_by_key << std::dec - << ", id:" << o.key + << ", id:" << o.key << ", data:" << o.blob << "}"; return out; } @@ -372,5 +376,5 @@ std::enable_if_t,std::is_s } **/ -} // namespace cascade -} // namespace derecho +} // namespace cascade +} // namespace derecho diff --git a/scripts/util/cascade_runner.sh b/scripts/util/cascade_runner.sh new file mode 100755 index 00000000..ecdcbde3 --- /dev/null +++ b/scripts/util/cascade_runner.sh @@ -0,0 +1,205 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_NAME=$(basename "${BASH_SOURCE[0]}") +HELP_STRING="A helper script to set up a local cascade instance +Usage: $SCRIPT_NAME [options] + -h Print this message and exit. + -a ARGS Args passed to resulting command. + -d D CFG directory location (defaults to cfg). + -n N Sets a range from 0..N-1 for range commands. + + -u K update key in range. + -v V update value in range. + (ex. -n 7 -u default_log_level -v info) + + -k Kill running cascade nodes. + -r Remove old logs in range. + -s Start server nodes in range. + -f D directory to mount (requires -c flag to start and -k to end). + -F D directory to mount (runs in foreground). + -c I Start client node at folder cfg/nI." + +CFG_DIR="$PWD/cfg" +FUSE_FOREGROUND=false +FUSE_DIR="" +HELP=false +KILL=false +CLEAN=false +SERVERS=false +NODES=0 +CLIENT=-1 + +UPDATE_KEY="" +UPDATE_VALUE="" + +# TODO get from env? or as arg +BASE_PATH="/root/workspace/cascade/build-Release/src" + +export PATH="$PATH:$BASE_PATH/service" +export PATH="$PATH:$BASE_PATH/service/fuse" + +FUSE_COMMAND="cascade_fuse_client_hl" # "cascade_fuse_client" +CLIENT_COMMAND="cascade_client" +SERVER_COMMAND="cascade_server" +LOG_FILE="node_info.log" + +ARGS="" + +if ! command -v $SERVER_COMMAND &>/dev/null; then + echo "$SERVER_COMMAND could not be found" + exit_abnormal +fi + +usage() { + echo "$HELP_STRING" 1>&2 +} + +exit_abnormal() { + usage + exit 1 +} + +check_num() { + # $1: string to check + RE_ISNUM='^[0-9]+$' + if ! [[ $1 =~ $RE_ISNUM ]]; then + exit_abnormal + fi +} + +foreach() { + # $1: command to do (accepts count as arg) + COUNT=0 + while [ $COUNT -lt "$NODES" ]; do + DIR="$CFG_DIR/n$COUNT" + cd "$DIR" + + $1 $COUNT + + cd - &>/dev/null + ((COUNT += 1)) + done +} + +main() { + while getopts 'ha:d:n:f:F:krsc:u:v:' opt; do + case "$opt" in + h) HELP=true ;; + a) ARGS="$OPTARG" ;; + d) CFG_DIR="$OPTARG" ;; + n) + NODES="$OPTARG" + check_num "$NODES" + ;; + f) FUSE_DIR="$OPTARG" ;; + F) + FUSE_DIR="$OPTARG" + FUSE_FOREGROUND=true + ;; + k) KILL=true ;; + r) CLEAN=true ;; + s) SERVERS=true ;; + c) + CLIENT="$OPTARG" + check_num "$CLIENT" + ;; + u) UPDATE_KEY="$OPTARG" ;; + v) UPDATE_VALUE="$OPTARG" ;; + *) exit_abnormal ;; + esac + done + + if [[ "$HELP" = true ]]; then + usage + exit + fi + + if [[ -n "$FUSE_DIR" ]]; then + MOUNT="$(realpath "$FUSE_DIR")" + + if [[ ! "$CLIENT" -eq "-1" ]]; then + DIR="$CFG_DIR/n$CLIENT" + cd "$DIR" + + echo "fuse client mounting at $MOUNT" + mkdir -p "$MOUNT" + + if [[ "$FUSE_FOREGROUND" = true ]]; then + "$FUSE_COMMAND" -s -f "$MOUNT" $ARGS + else + "$FUSE_COMMAND" -s -f "$MOUNT" $ARGS &>"$LOG_FILE" & + fi + + cd - &>/dev/null + exit + fi + + if [[ "$KILL" = true ]]; then + echo "unmounting $MOUNT and ending fuses client" + umount "$MOUNT" + + pkill -SIGINT "$FUSE_COMMAND" || true + + exit + fi + fi + + if [[ "$KILL" = true ]]; then + # TODO better idea: save PIDs? + + pkill -SIGINT "$CLIENT_COMMAND" || true + pkill -SIGINT "$SERVER_COMMAND" || true # kill cascade_server + + # kill old cascade_runner and subprocesses (excluding itself) + # pgrep "${SCRIPT_NAME%.*}" | grep -v "$$" | while read -r LINE; do + # SUB_SLEEP=$(pgrep -P "$LINE") + # kill "$LINE" # kill cascade_runner while loop + # kill "$SUB_SLEEP" # kill sleep + # done + fi + + if [[ "$CLEAN" = true ]]; then + x() { + echo "$1 > removing .plog and logs" + rm -rf .plog derecho_debug.log "$LOG_FILE" + } + foreach x + fi + + if [[ -n "$UPDATE_KEY" ]]; then + x() { + CONFIG_FILE="derecho.cfg" + echo "$1 > updating $UPDATE_KEY with $UPDATE_VALUE in $CONFIG_FILE" + sed -i "s/\($UPDATE_KEY *= *\)\(.*\)/\1$UPDATE_VALUE/" "$CONFIG_FILE" + } + foreach x + fi + + if [[ ! "$CLIENT" -eq "-1" ]]; then + DIR="$CFG_DIR/n$CLIENT" + + if [[ -z "$FUSE_DIR" ]]; then + cd "$DIR" + "$CLIENT_COMMAND" + fi + + cd - &>/dev/null + exit + fi + set +e + + if [[ "$SERVERS" = true ]]; then + x() { + echo "$1 > starting server node" + "$SERVER_COMMAND" --signal &>"$LOG_FILE" & + # (while true; do sleep 10000; done) | + # "$SERVER_COMMAND" &>"$LOG_FILE" & + } + foreach x + + fi +} + +main "$@" diff --git a/src/service/client.cpp b/src/service/client.cpp index c022b522..318d4370 100644 --- a/src/service/client.cpp +++ b/src/service/client.cpp @@ -1,20 +1,20 @@ #include -#include -#include +#include #include -#include -#include -#include +#include #include -#include +#include +#include +#include #include +#include #ifdef ENABLE_EVALUATION #include "perftest.hpp" -#endif//ENABLE_EVALUATION +#endif // ENABLE_EVALUATION using namespace derecho::cascade; -#define PROC_NAME "cascade_client" +#define PROC_NAME "cascade_client" template void print_subgroup_member(ServiceClientAPI& capi, uint32_t subgroup_index) { @@ -22,9 +22,9 @@ void print_subgroup_member(ServiceClientAPI& capi, uint32_t subgroup_index) { << "subgroup_index=" << subgroup_index << ")" << std::endl; auto members = capi.template get_subgroup_members(subgroup_index); uint32_t shard_index = 0; - for (const auto& shard: members) { + for(const auto& shard : members) { std::cout << "shard-" << shard_index << " = ["; - for (const auto& nid: shard) { + for(const auto& nid : shard) { std::cout << nid << ","; } std::cout << "]" << std::endl; @@ -36,9 +36,9 @@ void print_subgroup_member(ServiceClientAPI& capi, const std::string& op) { std::cout << "Object Pool=" << op << std::endl; auto members = capi.get_subgroup_members(op); uint32_t shard_index = 0; - for (const auto& shard: members) { + for(const auto& shard : members) { std::cout << "shard-" << shard_index << " = ["; - for (const auto& nid: shard) { + for(const auto& nid : shard) { std::cout << nid << ","; } std::cout << "]" << std::endl; @@ -50,9 +50,9 @@ template void print_shard_member(ServiceClientAPI& capi, uint32_t subgroup_index, uint32_t shard_index) { std::cout << "Subgroup (Type=" << std::type_index(typeid(SubgroupType)).name() << "," << "subgroup_index=" << subgroup_index << "," - << "shard_index=" << shard_index << ") member list = ["; - auto members = capi.template get_shard_members(subgroup_index,shard_index); - for (auto nid : members) { + << "shard_index=" << shard_index << ") member list = ["; + auto members = capi.template get_shard_members(subgroup_index, shard_index); + for(auto nid : members) { std::cout << nid << ","; } std::cout << "]" << std::endl; @@ -61,14 +61,13 @@ void print_shard_member(ServiceClientAPI& capi, uint32_t subgroup_index, uint32_ void print_shard_member(ServiceClientAPI& capi, const std::string& op, uint32_t shard_index) { std::cout << "Object Pool=" << op << ",\n" << "shard_index=" << shard_index << ",\nmember list=["; - auto members = capi.get_shard_members(op,shard_index); - for (auto nid : members) { + auto members = capi.get_shard_members(op, shard_index); + for(auto nid : members) { std::cout << nid << ","; } std::cout << "]" << std::endl; } - /** disabled void print_shard_member(ServiceClientAPI& capi, derecho::subgroup_id_t subgroup_id, uint32_t shard_index) { std::cout << "subgroup_id=" << subgroup_id << "," @@ -86,27 +85,26 @@ void print_shard_member(ServiceClientAPI& capi, derecho::subgroup_id_t subgroup_ * defined in include/cascade/service.hpp */ static const char* policy_names[] = { - "FirstMember", - "LastMember", - "Random", - "FixedRandom", - "RoundRobin", - "KeyHashing", - "UserSpecified", - nullptr -}; + "FirstMember", + "LastMember", + "Random", + "FixedRandom", + "RoundRobin", + "KeyHashing", + "UserSpecified", + nullptr}; inline ShardMemberSelectionPolicy parse_policy_name(const std::string& policy_name) { ShardMemberSelectionPolicy policy = ShardMemberSelectionPolicy::FirstMember; - int i=1; - while(policy_names[i]){ - if (policy_name == policy_names[i]) { + int i = 1; + while(policy_names[i]) { + if(policy_name == policy_names[i]) { policy = static_cast(i); break; } i++; } - if (policy_names[i] == nullptr) { + if(policy_names[i] == nullptr) { return ShardMemberSelectionPolicy::InvalidPolicy; } return policy; @@ -116,15 +114,15 @@ template void print_member_selection_policy(ServiceClientAPI& capi, uint32_t subgroup_index, uint32_t shard_index) { std::cout << "Subgroup (Type=" << std::type_index(typeid(SubgroupType)).name() << "," << "subgroup_index=" << subgroup_index << "," - << "shard_index=" << shard_index << ") policy="; - auto policy = capi.template get_member_selection_policy(subgroup_index,shard_index); + << "shard_index=" << shard_index << ") policy="; + auto policy = capi.template get_member_selection_policy(subgroup_index, shard_index); std::cout << policy_names[std::get<0>(policy)] << "(" << std::get<0>(policy) << ")," << std::get<1>(policy) << "" << std::endl; } template void set_member_selection_policy(ServiceClientAPI& capi, uint32_t subgroup_index, uint32_t shard_index, - ShardMemberSelectionPolicy policy, node_id_t user_specified_node_id) { - capi.template set_member_selection_policy(subgroup_index,shard_index,policy,user_specified_node_id); + ShardMemberSelectionPolicy policy, node_id_t user_specified_node_id) { + capi.template set_member_selection_policy(subgroup_index, shard_index, policy, user_specified_node_id); } /* TEST1: members */ @@ -132,25 +130,25 @@ void member_test(ServiceClientAPI& capi) { // print all members. std::cout << "Top Derecho group members = ["; auto members = capi.get_members(); - for (auto nid: members) { - std::cout << nid << "," ; + for(auto nid : members) { + std::cout << nid << ","; } std::cout << "]" << std::endl; // print per Subgroup Members: - print_shard_member(capi,0,0); - print_shard_member(capi,0,0); + print_shard_member(capi, 0, 0); + print_shard_member(capi, 0, 0); } -static std::vector tokenize(std::string &line, const char *delimiter) { +static std::vector tokenize(std::string& line, const char* delimiter) { std::vector tokens; char line_buf[1024]; std::strcpy(line_buf, line.c_str()); - char *token = std::strtok(line_buf, delimiter); - while (token != nullptr) { + char* token = std::strtok(line_buf, delimiter); + while(token != nullptr) { tokens.push_back(std::string(token)); token = std::strtok(NULL, delimiter); } - return tokens; // RVO + return tokens; // RVO } static void print_red(std::string msg) { @@ -165,30 +163,30 @@ static void print_cyan(std::string msg) { << "\033[0m" << std::endl; } -#define on_subgroup_type(x, ft, ...) \ - if ((x) == "VCSS") { \ - ft (__VA_ARGS__); \ - } else if ((x) == "PCSS") { \ - ft (__VA_ARGS__); \ - } else if ((x) == "TCSS") { \ - ft (__VA_ARGS__); \ - } else { \ - print_red("unknown subgroup type:" + x); \ +#define on_subgroup_type(x, ft, ...) \ + if((x) == "VCSS") { \ + ft(__VA_ARGS__); \ + } else if((x) == "PCSS") { \ + ft(__VA_ARGS__); \ + } else if((x) == "TCSS") { \ + ft(__VA_ARGS__); \ + } else { \ + print_red("unknown subgroup type:" + x); \ } -#define check_put_and_remove_result(result) \ - for (auto& reply_future:result.get()) {\ - auto reply = reply_future.second.get();\ - std::cout << "node(" << reply_future.first << ") replied with version:" << std::get<0>(reply)\ - << ",ts_us:" << std::get<1>(reply) << std::endl;\ +#define check_put_and_remove_result(result) \ + for(auto& reply_future : result.get()) { \ + auto reply = reply_future.second.get(); \ + std::cout << "node(" << reply_future.first << ") replied with version:" << std::get<0>(reply) \ + << ",ts_us:" << std::get<1>(reply) << std::endl; \ } template void put(ServiceClientAPI& capi, const std::string& key, const std::string& value, persistent::version_t pver, persistent::version_t pver_bk, uint32_t subgroup_index, uint32_t shard_index) { typename SubgroupType::ObjectType obj; - if constexpr (std::is_same::value) { - obj.key = static_cast(std::stol(key,nullptr,0)); - } else if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { + obj.key = static_cast(std::stol(key, nullptr, 0)); + } else if constexpr(std::is_same::value) { obj.key = key; } else { print_red(std::string("Unhandled KeyType:") + typeid(typename SubgroupType::KeyType).name()); @@ -204,9 +202,9 @@ void put(ServiceClientAPI& capi, const std::string& key, const std::string& valu template void put_and_forget(ServiceClientAPI& capi, const std::string& key, const std::string& value, persistent::version_t pver, persistent::version_t pver_bk, uint32_t subgroup_index, uint32_t shard_index) { typename SubgroupType::ObjectType obj; - if constexpr (std::is_same::value) { - obj.key = static_cast(std::stol(key,nullptr,0)); - } else if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { + obj.key = static_cast(std::stol(key, nullptr, 0)); + } else if constexpr(std::is_same::value) { obj.key = key; } else { print_red(std::string("Unhandled KeyType:") + typeid(typename SubgroupType::KeyType).name()); @@ -214,7 +212,7 @@ void put_and_forget(ServiceClientAPI& capi, const std::string& key, const std::s } obj.previous_version = pver; obj.previous_version_by_key = pver_bk; - obj.blob = Blob(reinterpret_cast(value.c_str()),value.length()); + obj.blob = Blob(reinterpret_cast(value.c_str()), value.length()); capi.template put_and_forget(obj, subgroup_index, shard_index); std::cout << "put done." << std::endl; } @@ -231,7 +229,7 @@ void op_put(ServiceClientAPI& capi, const std::string& key, const std::string& v void op_put_file(ServiceClientAPI& capi, const std::string& key, const std::string& filename, persistent::version_t pver, persistent::version_t pver_bk) { // get file size - std::ifstream value_file(filename,std::ios::binary); + std::ifstream value_file(filename, std::ios::binary); if(!value_file.good()) { dbg_default_error("Cannot open file:{} for read.", filename); throw std::runtime_error("Cannot open file:" + filename + "for read"); @@ -240,11 +238,11 @@ void op_put_file(ServiceClientAPI& capi, const std::string& key, const std::stri std::size_t file_size = value_file.tellg(); value_file.seekg(0); // message generator - blob_generator_func_t message_generator = [&value_file] (uint8_t* buffer, const std::size_t size) { - value_file.read(reinterpret_cast(buffer),size); + blob_generator_func_t message_generator = [&value_file](uint8_t* buffer, const std::size_t size) { + value_file.read(reinterpret_cast(buffer), size); return size; }; - ObjectWithStringKey obj(key,message_generator,file_size); + ObjectWithStringKey obj(key, message_generator, file_size); obj.previous_version = pver; obj.previous_version_by_key = pver_bk; derecho::rpc::QueryResults result = capi.put(obj); @@ -257,14 +255,14 @@ void op_put_and_forget(ServiceClientAPI& capi, const std::string& key, const std obj.key = key; obj.previous_version = pver; obj.previous_version_by_key = pver_bk; - obj.blob = Blob(reinterpret_cast(value.c_str()),value.length()); + obj.blob = Blob(reinterpret_cast(value.c_str()), value.length()); capi.put_and_forget(obj); std::cout << "put done." << std::endl; } void op_put_file_and_forget(ServiceClientAPI& capi, const std::string& key, const std::string& filename, persistent::version_t pver, persistent::version_t pver_bk) { // get file size - std::ifstream value_file(filename,std::ios::binary); + std::ifstream value_file(filename, std::ios::binary); if(!value_file.good()) { dbg_default_error("Cannot open file:{} for read.", filename); throw std::runtime_error("Cannot open file:" + filename + "for read"); @@ -273,11 +271,11 @@ void op_put_file_and_forget(ServiceClientAPI& capi, const std::string& key, cons std::size_t file_size = value_file.tellg(); value_file.seekg(0); // message generator - blob_generator_func_t message_generator = [&value_file] (uint8_t* buffer, const std::size_t size) { - value_file.read(reinterpret_cast(buffer),size); + blob_generator_func_t message_generator = [&value_file](uint8_t* buffer, const std::size_t size) { + value_file.read(reinterpret_cast(buffer), size); return size; }; - ObjectWithStringKey obj(key,message_generator,file_size); + ObjectWithStringKey obj(key, message_generator, file_size); obj.previous_version = pver; obj.previous_version_by_key = pver_bk; capi.put_and_forget(obj); @@ -300,16 +298,16 @@ void create_object_pool(ServiceClientAPI& capi, const std::string& id, uint32_t template void trigger_put(ServiceClientAPI& capi, const std::string& key, const std::string& value, uint32_t subgroup_index, uint32_t shard_index) { typename SubgroupType::ObjectType obj; - if constexpr (std::is_same::value) { - obj.key = static_cast(std::stol(key,nullptr,0)); - } else if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { + obj.key = static_cast(std::stol(key, nullptr, 0)); + } else if constexpr(std::is_same::value) { obj.key = key; } else { print_red(std::string("Unhandled KeyType:") + typeid(typename SubgroupType::KeyType).name()); return; } - obj.blob = Blob(reinterpret_cast(value.c_str()),value.length()); + obj.blob = Blob(reinterpret_cast(value.c_str()), value.length()); derecho::rpc::QueryResults result = capi.template trigger_put(obj, subgroup_index, shard_index); result.get(); @@ -320,7 +318,7 @@ void op_trigger_put(ServiceClientAPI& capi, const std::string& key, const std::s ObjectWithStringKey obj; obj.key = key; - obj.blob = Blob(reinterpret_cast(value.c_str()),value.length()); + obj.blob = Blob(reinterpret_cast(value.c_str()), value.length()); derecho::rpc::QueryResults result = capi.trigger_put(obj); result.get(); @@ -330,28 +328,29 @@ void op_trigger_put(ServiceClientAPI& capi, const std::string& key, const std::s template void collective_trigger_put(ServiceClientAPI& capi, const std::string& key, const std::string& value, uint32_t subgroup_index, std::vector nodes) { typename SubgroupType::ObjectType obj; - if constexpr (std::is_same::value) { - obj.key = static_cast(std::stol(key,nullptr,0)); - } else if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { + obj.key = static_cast(std::stol(key, nullptr, 0)); + } else if constexpr(std::is_same::value) { obj.key = key; } else { print_red(std::string("Unhandled KeyType:") + typeid(typename SubgroupType::KeyType).name()); return; } - obj.blob = Blob(reinterpret_cast(value.c_str()),value.length()); - std::unordered_map>> nodes_and_futures; - for (auto& nid: nodes) { - nodes_and_futures.emplace(nid,nullptr); + obj.blob = Blob(reinterpret_cast(value.c_str()), value.length()); + std::unordered_map>> nodes_and_futures; + for(auto& nid : nodes) { + nodes_and_futures.emplace(nid, nullptr); } capi.template collective_trigger_put(obj, subgroup_index, nodes_and_futures); - for (auto& kv: nodes_and_futures) { + for(auto& kv : nodes_and_futures) { kv.second.get(); std::cout << "Finish sending to node " << kv.first << std::endl; } std::cout << "collective_trigger_put is done." << std::endl; + std::cout << "collective_trigger_put is done." << std::endl; } template @@ -373,120 +372,120 @@ void op_remove(ServiceClientAPI& capi, const std::string& key) { check_put_and_remove_result(result); } -#define check_get_result(result) \ - for (auto& reply_future:result.get()) {\ - auto reply = reply_future.second.get();\ - std::cout << "node(" << reply_future.first << ") replied with value:" << reply << std::endl;\ +#define check_get_result(result) \ + for(auto& reply_future : result.get()) { \ + auto reply = reply_future.second.get(); \ + std::cout << "node(" << reply_future.first << ") replied with value:" << reply << std::endl; \ } template -void get(ServiceClientAPI& capi, const std::string& key, persistent::version_t ver, bool stable, uint32_t subgroup_index,uint32_t shard_index) { - if constexpr (std::is_same::value) { +void get(ServiceClientAPI& capi, const std::string& key, persistent::version_t ver, bool stable, uint32_t subgroup_index, uint32_t shard_index) { + if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get( - static_cast(std::stol(key,nullptr,0)),ver,stable,subgroup_index,shard_index); + static_cast(std::stol(key, nullptr, 0)), ver, stable, subgroup_index, shard_index); check_get_result(result); - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get( - key,ver,stable,subgroup_index,shard_index); + key, ver, stable, subgroup_index, shard_index); check_get_result(result); } } template void get_by_time(ServiceClientAPI& capi, const std::string& key, uint64_t ts_us, bool stable, uint32_t subgroup_index, uint32_t shard_index) { - if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get_by_time( - static_cast(std::stol(key,nullptr,0)),ts_us,stable,subgroup_index,shard_index); + static_cast(std::stol(key, nullptr, 0)), ts_us, stable, subgroup_index, shard_index); check_get_result(result); - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get_by_time( - key,ts_us,stable,subgroup_index,shard_index); + key, ts_us, stable, subgroup_index, shard_index); check_get_result(result); } } template void multi_get(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index) { - if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template multi_get( - static_cast(std::stol(key,nullptr,0)),subgroup_index,shard_index); + static_cast(std::stol(key, nullptr, 0)), subgroup_index, shard_index); check_get_result(result); - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template multi_get( - key,subgroup_index,shard_index); + key, subgroup_index, shard_index); check_get_result(result); } } template -void get_size(ServiceClientAPI& capi, const std::string& key, persistent::version_t ver, bool stable, uint32_t subgroup_index,uint32_t shard_index) { - if constexpr (std::is_same::value) { +void get_size(ServiceClientAPI& capi, const std::string& key, persistent::version_t ver, bool stable, uint32_t subgroup_index, uint32_t shard_index) { + if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get_size( - static_cast(std::stol(key,nullptr,0)),ver,stable,subgroup_index,shard_index); + static_cast(std::stol(key, nullptr, 0)), ver, stable, subgroup_index, shard_index); check_get_result(result); - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get_size( - key,ver,stable,subgroup_index,shard_index); + key, ver, stable, subgroup_index, shard_index); check_get_result(result); } } template void multi_get_size(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index) { - if constexpr ( std::is_same::value) { - derecho::rpc::QueryResults result = capi.template multi_get_size ( - static_cast(std::stol(key,nullptr,0)),subgroup_index,shard_index); + if constexpr(std::is_same::value) { + derecho::rpc::QueryResults result = capi.template multi_get_size( + static_cast(std::stol(key, nullptr, 0)), subgroup_index, shard_index); check_get_result(result); - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template multi_get_size( - key,subgroup_index,shard_index); + key, subgroup_index, shard_index); check_get_result(result); } } template void get_size_by_time(ServiceClientAPI& capi, const std::string& key, uint64_t ts_us, bool stable, uint32_t subgroup_index, uint32_t shard_index) { - if constexpr (std::is_same::value) { + if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get_size_by_time( - static_cast(std::stol(key,nullptr,0)),ts_us,stable,subgroup_index,shard_index); + static_cast(std::stol(key, nullptr, 0)), ts_us, stable, subgroup_index, shard_index); check_get_result(result); - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { derecho::rpc::QueryResults result = capi.template get_size_by_time( - key,ts_us,stable,subgroup_index,shard_index); + key, ts_us, stable, subgroup_index, shard_index); check_get_result(result); } } -#define check_list_keys_result(result) \ - for (auto& reply_future:result.get()) {\ - auto reply = reply_future.second.get();\ - std::cout << "Keys:" << std::endl;\ - for (auto& key:reply) {\ - std::cout << " " << key << std::endl;\ - }\ +#define check_list_keys_result(result) \ + for(auto& reply_future : result.get()) { \ + auto reply = reply_future.second.get(); \ + std::cout << "Keys:" << std::endl; \ + for(auto& key : reply) { \ + std::cout << " " << key << std::endl; \ + } \ } -#define check_op_list_keys_result(result)\ - std::cout << "Keys:" << std::endl;\ - for (auto& key:result) {\ - std::cout << " " << key << std::endl;\ +#define check_op_list_keys_result(result) \ + std::cout << "Keys:" << std::endl; \ + for(auto& key : result) { \ + std::cout << " " << key << std::endl; \ } template void multi_list_keys(ServiceClientAPI& capi, uint32_t subgroup_index, uint32_t shard_index) { - derecho::rpc::QueryResults> result = capi.template multi_list_keys(subgroup_index,shard_index); + derecho::rpc::QueryResults> result = capi.template multi_list_keys(subgroup_index, shard_index); check_list_keys_result(result); } template void list_keys(ServiceClientAPI& capi, persistent::version_t ver, bool stable, uint32_t subgroup_index, uint32_t shard_index) { - derecho::rpc::QueryResults> result = capi.template list_keys(ver,stable,subgroup_index,shard_index); + derecho::rpc::QueryResults> result = capi.template list_keys(ver, stable, subgroup_index, shard_index); check_list_keys_result(result); } template void list_keys_by_time(ServiceClientAPI& capi, uint64_t ts_us, bool stable, uint32_t subgroup_index, uint32_t shard_index) { - derecho::rpc::QueryResults> result = capi.template list_keys_by_time(ts_us,stable,subgroup_index,shard_index); + derecho::rpc::QueryResults> result = capi.template list_keys_by_time(ts_us, stable, subgroup_index, shard_index); check_list_keys_result(result); } @@ -495,13 +494,14 @@ void list_keys_by_time(ServiceClientAPI& capi, uint64_t ts_us, bool stable, uint template void list_data_by_prefix(ServiceClientAPI& capi, std::string prefix, persistent::version_t ver, uint32_t subgroup_index, uint32_t shard_index) { std::vector keys; - for (auto& obj : from_shard(keys,capi,subgroup_index,shard_index,ver).where([&prefix](typename SubgroupType::ObjectType o){ - if (o.blob.size < prefix.size()) { - return false; - } else { - return (std::string(reinterpret_cast(o.blob.bytes),prefix.size()) == prefix); - } - }).toStdVector()) { + for(auto& obj : from_shard(keys, capi, subgroup_index, shard_index, ver).where([&prefix](typename SubgroupType::ObjectType o) { + if(o.blob.size < prefix.size()) { + return false; + } else { + return (std::string(reinterpret_cast(o.blob.bytes), prefix.size()) == prefix); + } + }) + .toStdVector()) { std::cout << "Found:" << obj << std::endl; } } @@ -511,87 +511,90 @@ void list_data_by_prefix(ServiceClientAPI& c print_red("TCSS does not support list_data_by_prefix."); } - // "list_data_between_versions [version_begin] [version_end]\n\t test LINQ api - version_iterator \n" template -void list_data_between_versions(ServiceClientAPI &capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, persistent::version_t ver_begin, persistent::version_t ver_end) { - if constexpr (std::is_same::value) { - auto result = capi.template get(static_cast(std::stol(key,nullptr,0)), ver_end, subgroup_index, shard_index); - for (auto &reply_future : result.get()) { +void list_data_between_versions(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, persistent::version_t ver_begin, persistent::version_t ver_end) { + if constexpr(std::is_same::value) { + auto result = capi.template get(static_cast(std::stol(key, nullptr, 0)), ver_end, subgroup_index, shard_index); + for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); - if (reply.is_valid()) { + if(reply.is_valid()) { ver_end = reply.version; } else { return; } } - for (auto &obj : from_versions(static_cast(std::stol(key,nullptr,0)), capi, subgroup_index, shard_index, ver_end).where([ver_begin](typename SubgroupType::ObjectType obj) { - return ver_begin == INVALID_VERSION || obj.version >= ver_begin; - }).toStdVector()) { + for(auto& obj : from_versions(static_cast(std::stol(key, nullptr, 0)), capi, subgroup_index, shard_index, ver_end).where([ver_begin](typename SubgroupType::ObjectType obj) { + return ver_begin == INVALID_VERSION || obj.version >= ver_begin; + }) + .toStdVector()) { std::cout << "Found:" << obj << std::endl; } - } else if constexpr (std::is_same::value) { - auto result = capi.template get(key, ver_end, true/*always use stable here*/, subgroup_index, shard_index); - for (auto &reply_future : result.get()) { + } else if constexpr(std::is_same::value) { + auto result = capi.template get(key, ver_end, true /*always use stable here*/, subgroup_index, shard_index); + for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); - if (reply.is_valid()) { + if(reply.is_valid()) { ver_end = reply.version; } else { return; } } - for (auto &obj : from_versions(key, capi, subgroup_index, shard_index, ver_end).where([ver_begin](typename SubgroupType::ObjectType obj) { - return ver_begin == INVALID_VERSION || obj.version >= ver_begin; - }).toStdVector()) { + for(auto& obj : from_versions(key, capi, subgroup_index, shard_index, ver_end).where([ver_begin](typename SubgroupType::ObjectType obj) { + return ver_begin == INVALID_VERSION || obj.version >= ver_begin; + }) + .toStdVector()) { std::cout << "Found:" << obj << std::endl; } } } template <> -void list_data_between_versions(ServiceClientAPI &capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, persistent::version_t ver_begin, persistent::version_t ver_end) { +void list_data_between_versions(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, persistent::version_t ver_begin, persistent::version_t ver_end) { print_red("TCSS does not support list_data_between_versions."); } template -void list_data_between_timestamps(ServiceClientAPI &capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, uint64_t ts_begin, uint64_t ts_end) { +void list_data_between_timestamps(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, uint64_t ts_begin, uint64_t ts_end) { std::vector keys; - if constexpr (std::is_same::value) { - auto result = capi.template get(static_cast(std::stol(key,nullptr,0)), CURRENT_VERSION, subgroup_index, shard_index); - for (auto &reply_future : result.get()) { + if constexpr(std::is_same::value) { + auto result = capi.template get(static_cast(std::stol(key, nullptr, 0)), CURRENT_VERSION, subgroup_index, shard_index); + for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); - if (reply.is_valid()) { + if(reply.is_valid()) { ts_end = reply.timestamp_us >= ts_end ? ts_end : reply.timestamp_us; } else { return; } } - for (auto &obj : from_shard_by_time(keys, capi, subgroup_index, shard_index, ts_end).where([&key,ts_begin](typename SubgroupType::ObjectType obj) { - return !obj.is_null() && static_cast(std::stol(key,nullptr,0)) == obj.key && obj.timestamp_us >= ts_begin; - }).toStdVector()) { + for(auto& obj : from_shard_by_time(keys, capi, subgroup_index, shard_index, ts_end).where([&key, ts_begin](typename SubgroupType::ObjectType obj) { + return !obj.is_null() && static_cast(std::stol(key, nullptr, 0)) == obj.key && obj.timestamp_us >= ts_begin; + }) + .toStdVector()) { std::cout << "Found:" << obj << std::endl; } - } else if constexpr (std::is_same::value) { + } else if constexpr(std::is_same::value) { // set the timestamp to the latest update if ts_end > latest_ts - auto result = capi.template get(key, CURRENT_VERSION, true/*always stable version here*/, subgroup_index, shard_index); - for (auto &reply_future : result.get()) { + auto result = capi.template get(key, CURRENT_VERSION, true /*always stable version here*/, subgroup_index, shard_index); + for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); - if (reply.is_valid()) { + if(reply.is_valid()) { ts_end = reply.timestamp_us >= ts_end ? ts_end : reply.timestamp_us; } else { return; } } - for (auto &obj : from_shard_by_time(keys, capi, subgroup_index, shard_index, ts_end).where([&key,ts_begin](typename SubgroupType::ObjectType obj) { - return (!obj.is_null() && key == obj.key && obj.timestamp_us >= ts_begin); - }).toStdVector()) { + for(auto& obj : from_shard_by_time(keys, capi, subgroup_index, shard_index, ts_end).where([&key, ts_begin](typename SubgroupType::ObjectType obj) { + return (!obj.is_null() && key == obj.key && obj.timestamp_us >= ts_begin); + }) + .toStdVector()) { std::cout << "Found:" << obj << std::endl; } } } template <> -void list_data_between_timestamps(ServiceClientAPI &capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, uint64_t ts_begin, uint64_t ts_end) { +void list_data_between_timestamps(ServiceClientAPI& capi, const std::string& key, uint32_t subgroup_index, uint32_t shard_index, uint64_t ts_begin, uint64_t ts_end) { print_red("TCSS does not support list_data_between_timestamp."); } @@ -603,7 +606,7 @@ void list_data_in_subgroup(ServiceClientAPI& capi, uint32_t subgroup_index, pers std::unordered_map> shardidx_to_keys; - for (auto &obj : from_subgroup(shardidx_to_keys, shard_linq_list, capi, subgroup_index, version).toStdVector()) { + for(auto& obj : from_subgroup(shardidx_to_keys, shard_linq_list, capi, subgroup_index, version).toStdVector()) { std::cout << "Found:" << obj << std::endl; } } @@ -616,19 +619,19 @@ void list_data_in_subgroup(ServiceClientAPI& template void list_data_in_objectpool(ServiceClientAPI& capi, persistent::version_t version, const std::string& objpool_path) { std::vector keys; - for (auto &obj : from_objectpool(capi,keys,version,objpool_path).toStdVector()) { + for(auto& obj : from_objectpool(capi, keys, version, objpool_path).toStdVector()) { std::cout << "Found:" << obj << std::endl; } } -#endif// HAS_BOOLINQ +#endif // HAS_BOOLINQ // register notification template bool register_notification(ServiceClientAPI& capi, uint32_t subgroup_index) { return capi.template register_notification_handler( - [](const Blob& msg)->void{ + [](const Blob& msg) -> void { std::cout << "Subgroup Notification received:" - << "data:" << std::string(reinterpret_cast(msg.bytes),msg.size) + << "data:" << std::string(reinterpret_cast(msg.bytes), msg.size) << std::endl; }, subgroup_index); @@ -637,7 +640,7 @@ bool register_notification(ServiceClientAPI& capi, uint32_t subgroup_index) { // unregister notification template bool unregister_notification(ServiceClientAPI& capi, uint32_t subgroup_index) { - return capi.template register_notification_handler({},subgroup_index); + return capi.template register_notification_handler({}, subgroup_index); } #ifdef ENABLE_EVALUATION @@ -652,8 +655,8 @@ bool perftest(PerfTestClient& ptc, uint64_t duration_secs, const std::string& output_file) { debug_enter_func_with_args("put_type={},object_pool_pathname={},ec2cs={},read_write_ratio={},ops_threshold={},duration_secs={},output_file={}", - put_type,object_pool_pathname,static_cast(ec2cs),read_write_ratio,ops_threshold,duration_secs,output_file); - bool ret = ptc.template perf_put(put_type,object_pool_pathname,ec2cs,read_write_ratio,ops_threshold,duration_secs,output_file); + put_type, object_pool_pathname, static_cast(ec2cs), read_write_ratio, ops_threshold, duration_secs, output_file); + bool ret = ptc.template perf_put(put_type, object_pool_pathname, ec2cs, read_write_ratio, ops_threshold, duration_secs, output_file); debug_leave_func(); return ret; } @@ -670,64 +673,63 @@ bool perftest(PerfTestClient& ptc, uint64_t duration_secs, const std::string& output_file) { debug_enter_func_with_args("put_type={},subgroup_index={},shard_index={},ec2cs={},read_write_ratio={},ops_threshold={},duration_secs={},output_file={}", - put_type,subgroup_index, shard_index,static_cast(ec2cs),read_write_ratio,ops_threshold,duration_secs,output_file); - bool ret = ptc.template perf_put(put_type,subgroup_index,shard_index,ec2cs,read_write_ratio,ops_threshold,duration_secs,output_file); + put_type, subgroup_index, shard_index, static_cast(ec2cs), read_write_ratio, ops_threshold, duration_secs, output_file); + bool ret = ptc.template perf_put(put_type, subgroup_index, shard_index, ec2cs, read_write_ratio, ops_threshold, duration_secs, output_file); debug_leave_func(); return ret; } template -bool perftest_ordered_put(ServiceClientAPI &capi, +bool perftest_ordered_put(ServiceClientAPI& capi, uint32_t message_size, uint64_t duration_sec, uint32_t subgroup_index, - uint32_t shard_index){ + uint32_t shard_index) { debug_enter_func_with_args("message_size={},duration_sec={},subgroup_index={},shard_index={}.", - message_size,duration_sec,subgroup_index,shard_index); - auto result = capi.template perf_put(message_size,duration_sec,subgroup_index,shard_index); + message_size, duration_sec, subgroup_index, shard_index); + auto result = capi.template perf_put(message_size, duration_sec, subgroup_index, shard_index); check_get_result(result); debug_leave_func(); return true; } template <> -bool perftest_ordered_put(ServiceClientAPI &capi, - uint32_t message_size, - uint64_t duration_sec, - uint32_t subgroup_index, - uint32_t shard_index){ +bool perftest_ordered_put(ServiceClientAPI& capi, + uint32_t message_size, + uint64_t duration_sec, + uint32_t subgroup_index, + uint32_t shard_index) { print_red("TCSS does not support perftest_ordered_put"); return false; } template -bool dump_timestamp(ServiceClientAPI &capi, +bool dump_timestamp(ServiceClientAPI& capi, uint32_t subgroup_index, uint32_t shard_index, const std::string& filename) { debug_enter_func_with_args("subgroup_index={}, shard_index={}, filename={}", - subgroup_index,shard_index,filename); - auto result = capi.template dump_timestamp(filename,subgroup_index,shard_index); + subgroup_index, shard_index, filename); + auto result = capi.template dump_timestamp(filename, subgroup_index, shard_index); result.get(); TimestampLogger::flush(filename); debug_leave_func(); return true; } -#endif // ENABLE_EVALUATION - +#endif // ENABLE_EVALUATION /* TEST2: put/get/remove tests */ -using command_handler_t = std::function& cmd_tokens)>; +using command_handler_t = std::function& cmd_tokens)>; struct command_entry_t { - const std::string cmd; // command name - const std::string desc; // help info - const std::string help; // full help - const command_handler_t handler; // handler + const std::string cmd; // command name + const std::string desc; // help info + const std::string help; // full help + const command_handler_t handler; // handler }; void list_commands(const std::vector& command_list) { - for (const auto& entry: command_list) { - if (entry.handler) { + for(const auto& entry : command_list) { + if(entry.handler) { std::cout << std::left << std::setw(32) << entry.cmd << "- " << entry.desc << std::endl; } else { print_cyan("# " + entry.cmd + " #"); @@ -737,26 +739,29 @@ void list_commands(const std::vector& command_list) { ssize_t find_command(const std::vector& command_list, const std::string& command) { ssize_t pos = 0; - for(;pos < static_cast(command_list.size());pos++) { - if (command_list.at(pos).cmd == command) { + for(; pos < static_cast(command_list.size()); pos++) { + if(command_list.at(pos).cmd == command) { break; } } - if (pos == static_cast(command_list.size())) { + if(pos == static_cast(command_list.size())) { pos = -1; } return pos; } +void print_help(const std::string& cmd); bool shell_is_active = true; #define SUBGROUP_TYPE_LIST "VCSS|PCSS|TCSS" #define SHARD_MEMBER_SELECTION_POLICY_LIST "FirstMember|LastMember|Random|FixedRandom|RoundRobin|KeyHashing|UserSpecified" -#define CHECK_FORMAT(tks,argc) \ - if (tks.size() < argc) { \ - print_red("Invalid command format. Please try help " + tks[0] + "."); \ - return false; \ - } +#define CHECK_FORMAT(tks, argc) \ + if(tks.size() < argc) { \ + print_red("Invalid command format."); \ + print_help(tks[0]); \ + return false; \ + } + std::vector commands = { { @@ -1551,164 +1556,163 @@ std::vector commands = } }, #ifdef ENABLE_EVALUATION - { - "Performance Test Commands","","",command_handler_t() - }, - { - "perftest_object_pool", - "Performance Tester for put to an object pool.", - "perftest_object_pool [, ...] \n" - "type := " SUBGROUP_TYPE_LIST "\n" - "put_type := put|put_and_forget|trigger_put \n" - "'member selection policy' refers how the external clients pick a member in a shard;\n" - " Available options: FIXED|RANDOM|ROUNDROBIN;\n" - "'r/w ratio' is the ratio of get vs put operations, INF for all put test; \n" - "'max rate' is the maximum number of operations in Operations per Second, 0 for best effort; \n" - "'duration' is the span of the whole experiments; \n" - "'clientn' is a host[:port] pair representing the parallel clients. The port is default to " + std::to_string(PERFTEST_PORT), - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,9); - - PutType put_type = PutType::PUT; - - if (cmd_tokens[2] == "put_and_forget") { - put_type = PutType::PUT_AND_FORGET; - } else if (cmd_tokens[2] == "trigger_put") { - put_type = PutType::TRIGGER_PUT; - } - - std::string object_pool_pathname = cmd_tokens[3]; - ExternalClientToCascadeServerMapping member_selection_policy = FIXED; - if (cmd_tokens[4] == "RANDOM") { - member_selection_policy = ExternalClientToCascadeServerMapping::RANDOM; - } else if (cmd_tokens[4] == "ROUNDROBIN") { - member_selection_policy = ExternalClientToCascadeServerMapping::ROUNDROBIN; - } - double read_write_ratio = std::stod(cmd_tokens[5]); - uint64_t max_rate = std::stoul(cmd_tokens[6],nullptr,0); - uint64_t duration_sec = std::stoul(cmd_tokens[7],nullptr,0); - - PerfTestClient ptc{capi}; - uint32_t pos = 8; - while (pos < cmd_tokens.size()) { - std::string::size_type colon_pos = cmd_tokens[pos].find(':'); - if (colon_pos == std::string::npos) { - ptc.add_or_update_server(cmd_tokens[pos],PERFTEST_PORT); - } else { - ptc.add_or_update_server(cmd_tokens[pos].substr(0,colon_pos), - static_cast(std::stoul(cmd_tokens[pos].substr(colon_pos+1),nullptr,0))); - } - pos ++; - } - bool ret = false; - on_subgroup_type(cmd_tokens[1], ret = perftest, ptc, put_type, object_pool_pathname, member_selection_policy,read_write_ratio,max_rate,duration_sec,"timestamp.log"); - return ret; - } - }, - { - "perftest_shard", - "Performance Tester for put to a shard.", - "perftest_shard [, ...] \n" - "type := " SUBGROUP_TYPE_LIST "\n" - "put_type := put|put_and_forget|trigger_put \n" - "'member selection policy' refers how the external clients pick a member in a shard;\n" - " Available options: FIXED|RANDOM|ROUNDROBIN;\n" - "'r/w ratio' is the ratio of get vs put operations, INF for all put test; \n" - "'max rate' is the maximum number of operations in Operations per Second, 0 for best effort; \n" - "'duration' is the span of the whole experiments; \n" - "'clientn' is a host[:port] pair representing the parallel clients. The port is default to " + std::to_string(PERFTEST_PORT), - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,10); - - PutType put_type = PutType::PUT; - - if (cmd_tokens[2] == "put_and_forget") { - put_type = PutType::PUT_AND_FORGET; - } else if (cmd_tokens[2] == "trigger_put") { - put_type = PutType::TRIGGER_PUT; - } - - uint32_t subgroup_index = std::stoul(cmd_tokens[3],nullptr,0); - uint32_t shard_index = std::stoul(cmd_tokens[4],nullptr,0); - - ExternalClientToCascadeServerMapping member_selection_policy = FIXED; - if (cmd_tokens[5] == "RANDOM") { - member_selection_policy = ExternalClientToCascadeServerMapping::RANDOM; - } else if (cmd_tokens[5] == "ROUNDROBIN") { - member_selection_policy = ExternalClientToCascadeServerMapping::ROUNDROBIN; - } - double read_write_ratio = std::stod(cmd_tokens[6]); - uint64_t max_rate = std::stoul(cmd_tokens[7],nullptr,0); - uint64_t duration_sec = std::stoul(cmd_tokens[8],nullptr,0); - - PerfTestClient ptc{capi}; - uint32_t pos = 9; - while (pos < cmd_tokens.size()) { - std::string::size_type colon_pos = cmd_tokens[pos].find(':'); - if (colon_pos == std::string::npos) { - ptc.add_or_update_server(cmd_tokens[pos],PERFTEST_PORT); - } else { - ptc.add_or_update_server(cmd_tokens[pos].substr(0,colon_pos), - static_cast(std::stoul(cmd_tokens[pos].substr(colon_pos+1),nullptr,0))); - } - pos ++; - } - bool ret = false; - on_subgroup_type(cmd_tokens[1], ret = perftest,ptc,put_type,subgroup_index,shard_index,member_selection_policy,read_write_ratio,max_rate,duration_sec,"output.log"); - return ret; - } - }, - { - "perftest_ordered_put", - "Performance Test for ordered_put in a shard.", - "perftest_ordered_put \n" - "type := " SUBGROUP_TYPE_LIST "\n" - "'duration_sec' is the span of the whole experiments", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,6); - uint32_t message_size = std::stoul(cmd_tokens[2],nullptr,0); - uint64_t duration_sec = std::stoul(cmd_tokens[3],nullptr,0); - uint32_t subgroup_index = std::stoul(cmd_tokens[4],nullptr,0); - uint32_t shard_index = std::stoul(cmd_tokens[5],nullptr,0); - - on_subgroup_type(cmd_tokens[1], perftest_ordered_put, capi, message_size, duration_sec, subgroup_index, shard_index); - return true; - } - }, - { - "dump_timestamp", - "Dump timestamp for a given shard. Each node will write its timestamps to the given file.", - "dump_timestamp \n" - "type := " SUBGROUP_TYPE_LIST "\n" - "filename := timestamp log filename", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,5); - uint32_t subgroup_index = std::stoul(cmd_tokens[2],nullptr,0); - uint32_t shard_index = std::stoul(cmd_tokens[3],nullptr,0); - on_subgroup_type(cmd_tokens[1]/*subgroup type*/, dump_timestamp, capi, subgroup_index, shard_index, cmd_tokens[4]/*filename*/); - return true; - } - }, - { - "op_dump_timestamp", - "Dump timestamps for a given object pool. Each node will write its timestamps to the given file.", - "op_dump_timestamp \n" - "filename := timestamp log filename", - [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { - CHECK_FORMAT(cmd_tokens,3); - capi.dump_timestamp(cmd_tokens[2],cmd_tokens[1]); - TimestampLogger::flush(cmd_tokens[2]); - return true; - } - }, + {"Performance Test Commands", "", "", command_handler_t()}, + {"perftest_object_pool", + "Performance Tester for put to an object pool.", + "perftest_object_pool [, ...] \n" + "type := " SUBGROUP_TYPE_LIST "\n" + "put_type := put|put_and_forget|trigger_put \n" + "'member selection policy' refers how the external clients pick a member in a shard;\n" + " Available options: FIXED|RANDOM|ROUNDROBIN;\n" + "'r/w ratio' is the ratio of get vs put operations, INF for all put test; \n" + "'max rate' is the maximum number of operations in Operations per Second, 0 for best effort; \n" + "'duration' is the span of the whole experiments; \n" + "'clientn' is a host[:port] pair representing the parallel clients. The port is default to " + + std::to_string(PERFTEST_PORT), + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 9); + + PutType put_type = PutType::PUT; + + if(cmd_tokens[2] == "put_and_forget") { + put_type = PutType::PUT_AND_FORGET; + } else if(cmd_tokens[2] == "trigger_put") { + put_type = PutType::TRIGGER_PUT; + } + + std::string object_pool_pathname = cmd_tokens[3]; + ExternalClientToCascadeServerMapping member_selection_policy = FIXED; + if(cmd_tokens[4] == "RANDOM") { + member_selection_policy = ExternalClientToCascadeServerMapping::RANDOM; + } else if(cmd_tokens[4] == "ROUNDROBIN") { + member_selection_policy = ExternalClientToCascadeServerMapping::ROUNDROBIN; + } + double read_write_ratio = std::stod(cmd_tokens[5]); + uint64_t max_rate = std::stoul(cmd_tokens[6], nullptr, 0); + uint64_t duration_sec = std::stoul(cmd_tokens[7], nullptr, 0); + + PerfTestClient ptc{capi}; + uint32_t pos = 8; + while(pos < cmd_tokens.size()) { + std::string::size_type colon_pos = cmd_tokens[pos].find(':'); + if(colon_pos == std::string::npos) { + ptc.add_or_update_server(cmd_tokens[pos], PERFTEST_PORT); + } else { + ptc.add_or_update_server(cmd_tokens[pos].substr(0, colon_pos), + static_cast(std::stoul(cmd_tokens[pos].substr(colon_pos + 1), nullptr, 0))); + } + pos++; + } + bool ret = false; + on_subgroup_type(cmd_tokens[1], ret = perftest, ptc, put_type, object_pool_pathname, member_selection_policy, read_write_ratio, max_rate, duration_sec, "timestamp.log"); + return ret; + }}, + {"perftest_shard", + "Performance Tester for put to a shard.", + "perftest_shard [, ...] \n" + "type := " SUBGROUP_TYPE_LIST "\n" + "put_type := put|put_and_forget|trigger_put \n" + "'member selection policy' refers how the external clients pick a member in a shard;\n" + " Available options: FIXED|RANDOM|ROUNDROBIN;\n" + "'r/w ratio' is the ratio of get vs put operations, INF for all put test; \n" + "'max rate' is the maximum number of operations in Operations per Second, 0 for best effort; \n" + "'duration' is the span of the whole experiments; \n" + "'clientn' is a host[:port] pair representing the parallel clients. The port is default to " + + std::to_string(PERFTEST_PORT), + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 10); + + PutType put_type = PutType::PUT; + + if(cmd_tokens[2] == "put_and_forget") { + put_type = PutType::PUT_AND_FORGET; + } else if(cmd_tokens[2] == "trigger_put") { + put_type = PutType::TRIGGER_PUT; + } + + uint32_t subgroup_index = std::stoul(cmd_tokens[3], nullptr, 0); + uint32_t shard_index = std::stoul(cmd_tokens[4], nullptr, 0); + + ExternalClientToCascadeServerMapping member_selection_policy = FIXED; + if(cmd_tokens[5] == "RANDOM") { + member_selection_policy = ExternalClientToCascadeServerMapping::RANDOM; + } else if(cmd_tokens[5] == "ROUNDROBIN") { + member_selection_policy = ExternalClientToCascadeServerMapping::ROUNDROBIN; + } + double read_write_ratio = std::stod(cmd_tokens[6]); + uint64_t max_rate = std::stoul(cmd_tokens[7], nullptr, 0); + uint64_t duration_sec = std::stoul(cmd_tokens[8], nullptr, 0); + + PerfTestClient ptc{capi}; + uint32_t pos = 9; + while(pos < cmd_tokens.size()) { + std::string::size_type colon_pos = cmd_tokens[pos].find(':'); + if(colon_pos == std::string::npos) { + ptc.add_or_update_server(cmd_tokens[pos], PERFTEST_PORT); + } else { + ptc.add_or_update_server(cmd_tokens[pos].substr(0, colon_pos), + static_cast(std::stoul(cmd_tokens[pos].substr(colon_pos + 1), nullptr, 0))); + } + pos++; + } + bool ret = false; + on_subgroup_type(cmd_tokens[1], ret = perftest, ptc, put_type, subgroup_index, shard_index, member_selection_policy, read_write_ratio, max_rate, duration_sec, "output.log"); + return ret; + }}, + {"perftest_ordered_put", + "Performance Test for ordered_put in a shard.", + "perftest_ordered_put \n" + "type := " SUBGROUP_TYPE_LIST "\n" + "'duration_sec' is the span of the whole experiments", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 6); + uint32_t message_size = std::stoul(cmd_tokens[2], nullptr, 0); + uint64_t duration_sec = std::stoul(cmd_tokens[3], nullptr, 0); + uint32_t subgroup_index = std::stoul(cmd_tokens[4], nullptr, 0); + uint32_t shard_index = std::stoul(cmd_tokens[5], nullptr, 0); + + on_subgroup_type(cmd_tokens[1], perftest_ordered_put, capi, message_size, duration_sec, subgroup_index, shard_index); + return true; + }}, + {"dump_timestamp", + "Dump timestamp for a given shard. Each node will write its timestamps to the given file.", + "dump_timestamp \n" + "type := " SUBGROUP_TYPE_LIST "\n" + "filename := timestamp log filename", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 5); + uint32_t subgroup_index = std::stoul(cmd_tokens[2], nullptr, 0); + uint32_t shard_index = std::stoul(cmd_tokens[3], nullptr, 0); + on_subgroup_type(cmd_tokens[1] /*subgroup type*/, dump_timestamp, capi, subgroup_index, shard_index, cmd_tokens[4] /*filename*/); + return true; + }}, + {"op_dump_timestamp", + "Dump timestamps for a given object pool. Each node will write its timestamps to the given file.", + "op_dump_timestamp \n" + "filename := timestamp log filename", + [](ServiceClientAPI& capi, const std::vector& cmd_tokens) { + CHECK_FORMAT(cmd_tokens, 3); + capi.dump_timestamp(cmd_tokens[2], cmd_tokens[1]); + TimestampLogger::flush(cmd_tokens[2]); + return true; + }}, #endif }; +void print_help(const std::string& cmd) { + ssize_t command_index = find_command(commands, cmd); + if(command_index < 0) { + print_red("unknown command:'" + cmd + "'."); + } else { + print_red(commands.at(command_index).help); + } +} + inline void do_command(ServiceClientAPI& capi, const std::vector& cmd_tokens) { try { ssize_t command_index = find_command(commands, cmd_tokens[0]); - if (command_index>=0) { - if (commands.at(command_index).handler(capi,cmd_tokens)) { + if(command_index >= 0) { + if(commands.at(command_index).handler(capi, cmd_tokens)) { std::cout << "-> Succeeded." << std::endl; } else { std::cout << "-> Failed." << std::endl; @@ -1716,39 +1720,41 @@ inline void do_command(ServiceClientAPI& capi, const std::vector& c } else { print_red("unknown command:" + cmd_tokens[0]); } - } catch (const derecho::derecho_exception &ex) { - print_red (std::string("Exception:") + ex.what()); - } catch (...) { - print_red ("Unknown exception caught."); + } catch(const derecho::derecho_exception& ex) { + print_red(std::string("Exception:") + ex.what()); + } catch(...) { + print_red("Unknown exception caught."); } } void interactive_test(ServiceClientAPI& capi) { // loop - while (shell_is_active) { - char* malloced_cmd = readline("cmd> "); + while(shell_is_active) { + char* malloced_cmd = readline("\033[1;36m" + "cmd> " + "\033[0m"); std::string cmdline(malloced_cmd); free(malloced_cmd); - if (cmdline == "")continue; + if(cmdline == "") continue; add_history(cmdline.c_str()); std::string delimiter = " "; - do_command(capi,tokenize(cmdline, delimiter.c_str())); + do_command(capi, tokenize(cmdline, delimiter.c_str())); } std::cout << "Client exits." << std::endl; } void detached_test(ServiceClientAPI& capi, int argc, char** argv) { std::vector cmd_tokens; - for(int i=1;i + $ + $ + ) + + add_executable(fuse_client_hl fuse_client_hl.cpp) + target_include_directories(fuse_client_hl PRIVATE + $ + $ + $ + $ + ) + target_link_libraries(fuse_client_hl cascade readline fuse3 fuse_client_signals) + set_target_properties(fuse_client_hl PROPERTIES OUTPUT_NAME cascade_fuse_client_hl) + add_executable(fuse_client fuse_client.cpp) target_include_directories(fuse_client PRIVATE $ @@ -13,9 +30,9 @@ if (${HAS_FUSE}) $ $ ) - target_link_libraries(fuse_client cascade readline fuse3) + target_link_libraries(fuse_client cascade readline fuse3 fuse_client_signals) set_target_properties(fuse_client PROPERTIES OUTPUT_NAME cascade_fuse_client) - + add_custom_command(TARGET fuse_client POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/testfuse.py ${CMAKE_CURRENT_BINARY_DIR}/. COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/util.py ${CMAKE_CURRENT_BINARY_DIR}/. @@ -29,4 +46,3 @@ if (${HAS_FUSE}) ) endif() - diff --git a/src/service/fuse/README.md b/src/service/fuse/README.md index a5da1b86..8fcb79ba 100644 --- a/src/service/fuse/README.md +++ b/src/service/fuse/README.md @@ -1,12 +1,24 @@ # Fuse API -Cascade service supports FUSE(Filesystem in Userspace) API to access data. It is a standalone client application that links with [`libfuse`](https://github.com/libfuse/libfuse). Cascade fuse api mount the file system to interact with Cascade K/V stored objects. +Cascade service supports FUSE(Filesystem in Userspace) API to access data. It is +a standalone client application that links with +[`libfuse`](https://github.com/libfuse/libfuse). Cascade fuse api mount the file +system to interact with Cascade K/V stored objects. -If you didn't see `-- Looking for include files fuse3/fuse.h, fuse3/fuse_lowlevel.h - found`, it is probably because the libfuse support is not installed. FUSE application shall be built as an executable in the built directory `src/service/fuse/cascade_fuse_client` +If you didn't see +`-- Looking for include files fuse3/fuse.h, fuse3/fuse_lowlevel.h - found`, it +is probably because the libfuse support is not installed. FUSE application shall +be built as an executable in the built directory +`src/service/fuse/cascade_fuse_client` -Running FUSE POSIX shell application is similar to run cascade cmd client. Once the cascade service is configured and started, in the client node, running `../../fuse/cascade_fuse_client [directory_to_mount]` will mount cascade file systsem to the dedicated directory. +Running FUSE POSIX shell application is similar to run cascade cmd client. Once +the cascade service is configured and started, in the client node, running +`../../fuse/cascade_fuse_client [directory_to_mount]` will mount cascade file +systsem to the dedicated directory. -Once fuse applicatino is mounted to the directory, you can access the K/V object stored in cascade using linux shell command. The structured of the mounted file system is as following +Once fuse application is mounted to the directory, you can access the K/V object +stored in cascade using linux shell command. The structured of the mounted file +system is as following ```bash . @@ -14,36 +26,39 @@ Once fuse applicatino is mounted to the directory, you can access the K/V object |-- PersistentCascadeStoreWithStringKey | |-- subgroup-0 | |-- shard-0 -| |-- .cascade +| |-- .cascade | |-- key-0 | ... |-- VolatileCascadeStoreWithStringKey | |-- subgroup-0 | |-- shard-0 -| |-- .cascade +| |-- .cascade | ... |-- TriggerCascadeStoreWithStringKey |-- ObjectPools | |-- objectpoolpathname-a -| |-- a1 -| |-- objectpool object 0 -| ... -| |-- .cascade +| |-- a1 +| |-- objectpool object 0 +| ... +| |-- .cascade | |-- .cascade |-- .cascade ``` Support READ commands: + ``` -cd [dir] open directory -ls [dir] list directory -ls -a list directory with attributes -cat [file] read file -cat .cascade read directory metadata information +cd [dir] open directory +ls [dir] list directory +ls -a list directory with attributes +cat [file] read file +cat .cascade read directory metadata information ``` Limitation: + - Support only single-threaded fuse client -- Current read_file keeps a buffer of file_bytes in memory, needs further optimization to read large file +- Current read_file keeps a buffer of file_bytes in memory, needs further + optimization to read large file - New features to come: WRITE commands to have fuse client interact with cascade - File write and editing commands, managing directories commands \ No newline at end of file + File write and editing commands, managing directories commands diff --git a/src/service/fuse/fcc_hl.hpp b/src/service/fuse/fcc_hl.hpp new file mode 100644 index 00000000..44d99308 --- /dev/null +++ b/src/service/fuse/fcc_hl.hpp @@ -0,0 +1,479 @@ +#pragma once + +#define FUSE_USE_VERSION 34 + +#include +#include +#include +#include +#include +#include +#include + +#include "path_tree.hpp" + +namespace fs = std::filesystem; + +std::shared_ptr DL; + +enum NodeFlag : uint32_t { + ROOT_DIR = 1 << 0, + + OP_PREFIX_DIR = 1 << 1, + OP_ROOT_DIR = 1 << 2, + + KEY_DIR = 1 << 3, + KEY_FILE = 1 << 4, + + LATEST_DIR = 1 << 5, + + METADATA_PREFIX_DIR = 1 << 6, + METADATA_INFO_FILE = 1 << 7, + + SNAPSHOT_ROOT_DIR = 1 << 8, + SNAPSHOT_TIME_DIR = 1 << 9, +}; + +/* +op_get_by_time +op_get_size <-- create new function for content size ??? +VT template type ?? +op_list_keys_by_time +*/ + +const uint32_t FILE_FLAG = KEY_FILE | METADATA_INFO_FILE; +const uint32_t DIR_FLAG = ~FILE_FLAG; +const uint32_t OP_FLAG = OP_PREFIX_DIR | OP_ROOT_DIR | KEY_DIR | KEY_FILE; + +struct NodeData { + NodeFlag flag; + // timestamp in microsec + uint64_t timestamp; + std::vector bytes; + + bool writeable; + + NodeData(const NodeFlag flag) : flag(flag), timestamp(0), writeable(false) { + } +}; + +namespace derecho { +namespace cascade { + +struct FuseClientContext { + using Node = PathTree; + + std::unique_ptr root; + + // object pools are not versioned? :( + const fs::path METADATA_PATH = "/.cascade"; + const fs::path SNAPSHOT_PATH = "/snapshot"; + const fs::path LATEST_PATH = "/latest"; + const fs::path ROOT = "/"; + ServiceClientAPI& capi; + + bool version_snapshot; + persistent::version_t max_ver; + uint64_t max_timestamp; // timestamp in microsecond + + std::set local_latest_dirs; + std::set snapshots; + std::set snapshots_by_time; + + time_t update_interval; + time_t last_update_sec; + + FuseClientContext(int update_int, bool ver_snap) : capi(ServiceClientAPI::get_service_client()) { + DL = LoggerFactory::createLogger("fuse_client", spdlog::level::from_str(derecho::getConfString(CONF_LOGGER_DEFAULT_LOG_LEVEL))); + DL->set_pattern("[%T][%n][%^%l%$] %v"); + + max_ver = 0; + max_timestamp = 0; + + version_snapshot = ver_snap; + dbg_info(DL, "snapshot type: {}", version_snapshot ? "version" : "timestamp"); + update_interval = update_int; + last_update_sec = 0; + + root = std::make_unique(ROOT, NodeData(ROOT_DIR)); + root->set(SNAPSHOT_PATH, NodeData(SNAPSHOT_ROOT_DIR), NodeData(SNAPSHOT_ROOT_DIR)); + reset_latest(); + + update_object_pools(); + } + + /* --- pathtree related logic --- */ + + void reset_latest() { + auto latest = root->get(LATEST_PATH); + if(latest != nullptr) { + latest->data = NodeData(LATEST_DIR); + latest->children.clear(); + } else { + root->set(LATEST_PATH, NodeData(LATEST_DIR), NodeData(LATEST_DIR)); + } + } + + Node* add_op_info(const fs::path& path, const std::string& contents) { + auto node = root->set(path, NodeData(METADATA_PREFIX_DIR), + NodeData(METADATA_INFO_FILE)); + if(node == nullptr) { + return nullptr; + } + node->data.bytes = std::vector(contents.begin(), contents.end()); + return node; + } + + Node* add_snapshot_time(const fs::path& path) { + return root->set(path, NodeData(SNAPSHOT_ROOT_DIR), NodeData(SNAPSHOT_TIME_DIR)); + } + + Node* add_op_root(const fs::path& path) { + return root->set(path, NodeData(OP_PREFIX_DIR), NodeData(OP_ROOT_DIR)); + } + + Node* add_op_key(const fs::path& path) { + // invariant: assumes op_root already exists + return root->set(path, NodeData(KEY_DIR), NodeData(KEY_FILE)); + } + + Node* add_op_key_dir(const fs::path& path) { + // invariant: assumes op_root already exists + return root->set(path, NodeData(KEY_DIR), NodeData(KEY_DIR)); + } + + Node* object_pool_root(Node* node) { + if(node == nullptr) { + return nullptr; + } + if(node->parent == nullptr || node->data.flag & OP_ROOT_DIR) { + return node; + } + if(node->data.flag & (KEY_DIR | KEY_FILE)) { + return object_pool_root(node->parent); + } + return nullptr; + } + + Node* nearest_object_pool_root(const fs::path& path) { + auto node = root->get_while_valid(path); + return object_pool_root(node); + } + + bool add_snapshot(const fs::path& path) { + if(path.parent_path() != SNAPSHOT_PATH) { + return false; + } + try { + unsigned long long ts_us = std::stoull(path.filename()); + persistent::version_t ver = ts_us; + if(version_snapshot && ver <= max_ver) { + add_snapshot_folder(ver); + return true; + } + if(!version_snapshot && ts_us <= max_timestamp) { + add_snapshot_folder_by_time(ts_us); + return true; + } + } catch(std::invalid_argument const& ex) { + } catch(std::out_of_range const& ex) { + } + return false; + } + + void add_snapshot_folder(persistent::version_t ver) { + auto snapshot = SNAPSHOT_PATH; + snapshot += "/" + std::to_string(ver); + auto res = add_snapshot_time(snapshot); + dbg_info(DL, "adding {}", snapshot); + if(res != nullptr) { + fill_at(snapshot, ver); + } + } + + void add_snapshot_folder_by_time(uint64_t ts_us) { + auto snapshot = SNAPSHOT_PATH; + snapshot += "/" + std::to_string(ts_us); + auto res = add_snapshot_time(snapshot); + dbg_info(DL, "adding {}", snapshot); + if(res != nullptr) { + fill_at_by_time(snapshot, ts_us); + } + } + + /* --- capi related logic --- */ + + bool should_update() { + if(time(0) > last_update_sec + update_interval) { + return true; + } + return false; + } + + std::string path_while_op(const Node* node) const { + std::vector parts; + for(; node != nullptr && (node->data.flag & OP_FLAG); node = node->parent) { + parts.push_back(node->label); + } + std::string res; + for(auto it = parts.rbegin(); it != parts.rend(); ++it) { + res += "/" + *it; + } + + return res; + } + + int put_to_capi(const Node* node) { + // invariant: node is file + ObjectWithStringKey obj; + obj.key = path_while_op(node); + obj.previous_version = INVALID_VERSION; + obj.previous_version_by_key = INVALID_VERSION; + obj.blob = Blob(node->data.bytes.data(), node->data.bytes.size(), true); + // TODO verify emplaced avoids blob deleting data :( + + auto result = capi.put(obj); + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + dbg_info(DL, "node({}) replied with version:{},ts_us:{}", + reply_future.first, std::get<0>(reply), std::get<1>(reply)); + } + // TODO check for error + + return 0; + } + + void fill_op_meta(const fs::path& prefix, const std::string& op_root) { + auto opm = capi.find_object_pool(op_root); + json j{{"valid", opm.is_valid()}, + {"null", opm.is_null()}}; + if(opm.is_valid() && !opm.is_null()) { + j.emplace("pathname", opm.pathname); + j.emplace("version", opm.version); + j.emplace("timestamp_us", opm.timestamp_us); + j.emplace("previous_version", opm.previous_version); + j.emplace("previous_version_by_key", opm.previous_version_by_key); + j.emplace("subgroup_type", std::to_string(opm.subgroup_type_index) + "-->" + DefaultObjectPoolMetadataType::subgroup_type_order[opm.subgroup_type_index].name()); + j.emplace("subgroup_index", opm.subgroup_index); + j.emplace("sharding_policy", std::to_string(opm.sharding_policy)); + j.emplace("deleted", opm.deleted); + } + auto op_root_meta_path = prefix; + op_root_meta_path += METADATA_PATH; + op_root_meta_path += op_root; + add_op_info(op_root_meta_path, j.dump(2)); + } + + void fill_at(const fs::path& prefix, persistent::version_t ver) { + // TODO no version for list object pools + auto str_paths = capi.list_object_pools(false, true); + + if(ver == CURRENT_VERSION) { + auto meta_path = prefix; + meta_path += METADATA_PATH; + root->set(meta_path, + NodeData(METADATA_PREFIX_DIR), NodeData(METADATA_PREFIX_DIR)); + } + for(const std::string& op_root : str_paths) { + auto op_root_path = prefix; + op_root_path += op_root; + auto op_root_node = add_op_root(op_root_path); + if(op_root_node == nullptr) { + continue; + } + + if(ver == CURRENT_VERSION) { + fill_op_meta(prefix, op_root); + } + + auto keys = get_keys(op_root, ver); + std::sort(keys.begin(), keys.end(), std::greater<>()); + // sort removes files colliding with directory + for(const auto& k : keys) { + auto key_path = prefix; + key_path += k; + auto node = add_op_key(key_path); + // colliding keys do not get added + if(node != nullptr) { + get_contents(node, k, ver); + // dbg_info(DL, "file: {}", std::quoted(reinterpret_cast(node->data.bytes.data()))); + } + } + } + } + + void fill_at_by_time(const fs::path& prefix, uint64_t ts_us) { + // TODO no version for list object pools + auto str_paths = capi.list_object_pools(false, true); + + for(const std::string& op_root : str_paths) { + auto op_root_path = prefix; + op_root_path += op_root; + auto op_root_node = add_op_root(op_root_path); + if(op_root_node == nullptr) { + continue; + } + + auto keys = get_keys_by_time(op_root, ts_us); + std::sort(keys.begin(), keys.end(), std::greater<>()); + // sort removes files colliding with directory + for(const auto& k : keys) { + auto key_path = prefix; + key_path += k; + // colliding keys do not get added + auto node = add_op_key(key_path); + if(node != nullptr) { + get_contents_by_time(node, k, ts_us); + // dbg_info(DL, "file: {}", std::quoted(reinterpret_cast(node->data.bytes.data()))); + } + } + } + } + + void update_object_pools() { + // TODO use old cached data + reset_latest(); + + fill_at(LATEST_PATH, CURRENT_VERSION); + + for(auto it = local_latest_dirs.begin(); it != local_latest_dirs.end();) { + if(nearest_object_pool_root(*it) == nullptr) { + // TODO verify for op_root deleted + it = local_latest_dirs.erase(it); + } else { + add_op_key_dir(*it); + ++it; + } + } + + dbg_info(DL, "updating contents\n{}", string()); + + last_update_sec = time(0); + } + + int get_stat(Node* node, struct stat* stbuf) { + if(node == nullptr) { + return -ENOENT; + } + // not needed: st_dev, st_blksize, st_ino (unless use_ino mount option) + // TODO maintain stbuf in data? + stbuf->st_nlink = 1; + stbuf->st_uid = fuse_get_context()->uid; + stbuf->st_gid = fuse_get_context()->gid; + // TODO merge timestamp for dirs, update during set + int64_t sec = node->data.timestamp / 1'000'000; + int64_t nano = (node->data.timestamp % 1'000'000) * 1000; + stbuf->st_mtim = timespec{sec, nano}; + stbuf->st_ctim = stbuf->st_mtim; + stbuf->st_atim = timespec{last_update_sec, 0}; + if(uint64_t(last_update_sec) * 1'000'000 < node->data.timestamp) { + stbuf->st_atim = stbuf->st_mtim; + } + // - at prefix dir location add .info file ??? + + // TODO timestamps messing with vim?? + + if(node->data.flag & DIR_FLAG) { + if(node->data.flag & OP_PREFIX_DIR) { + stbuf->st_mode = S_IFDIR | 0555; + } else { + stbuf->st_mode = S_IFDIR | 0755; + } + stbuf->st_nlink = 2; // TODO calculate properly + for(const auto& [_, v] : node->children) { + stbuf->st_nlink += v->data.flag & DIR_FLAG; + } + } else { + // TODO somehow even when 0444, can still write ??? + stbuf->st_mode = S_IFREG | (node->data.flag & KEY_FILE ? 0744 : 0444); + stbuf->st_size = node->data.bytes.size(); + } + + // dev_t st_dev; /* ID of device containing file */ + // ino_t st_ino; /* Inode number */ + // mode_t st_mode; /* File type and mode */ + // nlink_t st_nlink; /* Number of hard links */ + // uid_t st_uid; /* User ID of owner */ + // gid_t st_gid; /* Group ID of owner */ + // dev_t st_rdev; /* Device ID (if special file) */ + // off_t st_size; /* Total size, in bytes */ + // blksize_t st_blksize; /* Block size for filesystem I/O */ + // blkcnt_t st_blocks; /* Number of 512B blocks allocated */ + + // struct timespec st_atim; /* Time of last access */ + // struct timespec st_mtim; /* Time of last modification */ + // struct timespec st_ctim; /* Time of last status change */ + return 0; + } + + std::string string() { + std::stringstream ss; + root->print(100, ss); + return ss.str(); + } + + void get_contents(Node* node, const std::string& path, persistent::version_t ver) { + auto result = capi.get(path, ver, true); + // TODO only get file contents on open + + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + if(ver == CURRENT_VERSION) { + max_ver = std::max(max_ver, reply.version); + max_timestamp = std::max(max_timestamp, reply.timestamp_us); + + node->data.writeable = true; + } + // TODO std::move ?? + Blob blob = reply.blob; + std::vector bytes(blob.bytes, blob.bytes + blob.size); + node->data.bytes = bytes; + node->data.timestamp = reply.timestamp_us; + return; + } + } + + // not to be called by latest + void get_contents_by_time(Node* node, const std::string& path, uint64_t ts_us) { + auto result = capi.get_by_time(path, ts_us, true); + // TODO only get file contents on open + + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + Blob blob = reply.blob; + // TODO std::move ?? + std::vector bytes(blob.bytes, blob.bytes + blob.size); + node->data.bytes = bytes; + node->data.timestamp = reply.timestamp_us; + return; + } + } + + std::vector get_keys(const std::string& path, persistent::version_t ver) { + auto future_result = capi.list_keys(ver, true, path); + return capi.wait_list_keys(future_result); + } + + std::vector get_keys_by_time(const std::string& path, uint64_t ts_us) { + auto future_result = capi.list_keys_by_time(ts_us, true, path); + return capi.wait_list_keys(future_result); + } + + // make a trash folder? (move on delete) + + Node* get(const std::string& path) { + if(should_update()) { + update_object_pools(); + // split between update object pool list and update keys + } + return root->get(path); + } + // TODO use object pool root meta file to edit version # and such? + + // TODO cascade metaservice api. need: + // - get children given path. shouldnt be hard considering op_list_keys works from subdir of op root + // - get file metadata WITHOUT getting file contents (file size, modification time) +}; + +} // namespace cascade +} // namespace derecho diff --git a/src/service/fuse/fuse_client.cpp b/src/service/fuse/fuse_client.cpp index b0dc45d8..a5316647 100644 --- a/src/service/fuse/fuse_client.cpp +++ b/src/service/fuse/fuse_client.cpp @@ -1,24 +1,25 @@ -#define FUSE_USE_VERSION 31 -#include -#include -#include -#include -#include -#include -#include -#include "fuse_client_context.hpp" #include #include #include +#include +#include +#include +#include +#include +#include + +#include "fuse_client_context.hpp" +#include "fuse_client_signals.hpp" -#include -#define FUSE_CLIENT_DEV_ID (0xCA7CADE) +#define FUSE_CLIENT_DEV_ID (0xCA7CADE) /** - * fuse_client mount the cascade service to file system. This allows users to access cascade data with normal POSIX + * fuse_client mounts the cascade service as a file system. This allows users to access cascade data with normal POSIX * filesystem API. - * + * + * TODO document data path for object pool + * * The data in cascade is organized this way: * ///// * "mount-point" is where the cascade data is mounted. @@ -33,160 +34,282 @@ using namespace derecho::cascade; using FuseClientContextType = FuseClientContext; -#define FCC(p) static_cast(p) -#define FCC_REQ(req) FCC(fuse_req_userdata(req)) +// #define FCC(p) static_cast(p) +// #define fcc(req) FCC(fuse_req_userdata(req)) + +static FuseClientContextType* fcc(fuse_req_t req) { + return static_cast(fuse_req_userdata(req)); +} + +static void fs_init(void* userdata, struct fuse_conn_info* conn) { + dbg_default_trace("entering {}.", __func__); + auto fcc = static_cast(userdata); + // TODO look through conn->capable options -static void fs_init(void* userdata, struct fuse_conn_info *conn) { - dbg_default_trace("entering {}.",__func__); - if (derecho::hasCustomizedConfKey(CONF_LAYOUT_JSON_LAYOUT)) { - FCC(userdata)->initialize(json::parse(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT))); - } else if (derecho::hasCustomizedConfKey(CONF_LAYOUT_JSON_LAYOUT_FILE)){ + if(derecho::hasCustomizedConfKey(CONF_LAYOUT_JSON_LAYOUT)) { + fcc->initialize(json::parse(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT))); + } else if(derecho::hasCustomizedConfKey(CONF_LAYOUT_JSON_LAYOUT_FILE)) { nlohmann::json layout_array; std::ifstream json_file(derecho::getAbsoluteFilePath(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT_FILE))); - if (!json_file) { - dbg_default_error("Cannot load json configuration from file: {}", derecho::getAbsoluteFilePath(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT_FILE))); + if(!json_file) { + dbg_default_error("Cannot load json configuration from file: {}", derecho::getAbsoluteFilePath(derecho::getConfString(CONF_LAYOUT_JSON_LAYOUT_FILE))); throw derecho::derecho_exception("Cannot load json configuration from file."); } json_file >> layout_array; - FCC(userdata)->initialize(layout_array); + fcc->initialize(layout_array); } - dbg_default_trace("leaving {}.",__func__); + dbg_default_trace("leaving {}.", __func__); } static void fs_destroy(void* userdata) { - dbg_default_trace("entering {}.",__func__); - dbg_default_trace("leaving {}.",__func__); + dbg_default_trace("entering {}.", __func__); + dbg_default_trace("leaving {}.", __func__); +} + +static int do_lookup(fuse_req_t req, fuse_ino_t parent, const char* name, + struct fuse_entry_param* e) { + // TODO: make this more efficient by implement a dedicated call in FCC. + auto name_to_ino = fcc(req)->get_dir_entries(parent); + if(name_to_ino.find(name) == name_to_ino.end()) { + return ENOENT; + } + // TODO: change timeout settings. + e->ino = name_to_ino.at(name); + e->attr_timeout = 10000.0; + e->entry_timeout = 10000.0; + e->attr.st_ino = e->ino; + fcc(req)->fill_stbuf_by_ino(e->attr); + + return 0; } static void fs_lookup(fuse_req_t req, fuse_ino_t parent, const char* name) { - dbg_default_trace("entering {}.",__func__); + dbg_default_trace("entering {}.", __func__); struct fuse_entry_param e; - // TODO: make this more efficient by implement a dedicated call in FCC. - auto name_to_ino = FCC_REQ(req)->get_dir_entries(parent); - if (name_to_ino.find(name) == name_to_ino.end()) { - fuse_reply_err(req, ENOENT); + + int err = do_lookup(req, parent, name, &e); + if(err) { + fuse_reply_err(req, err); } else { - // TODO: change timeout settings. - e.ino = name_to_ino.at(name); - e.attr_timeout = 10000.0; - e.entry_timeout = 10000.0; - e.attr.st_ino = e.ino; - FCC_REQ(req)->fill_stbuf_by_ino(e.attr); fuse_reply_entry(req, &e); } - dbg_default_trace("leaving {}.",__func__); + // auto name_to_ino = fcc(req)->get_dir_entries(parent); + // if(name_to_ino.find(name) == name_to_ino.end()) { + // fuse_reply_err(req, ENOENT); + // } else { + // // TODO: change timeout settings. + // e.ino = name_to_ino.at(name); + // e.attr_timeout = 10000.0; + // e.entry_timeout = 10000.0; + // e.attr.st_ino = e.ino; + // fcc(req)->fill_stbuf_by_ino(e.attr); + // fuse_reply_entry(req, &e); + // } + + dbg_default_trace("leaving {}.", __func__); } static void fs_getattr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { - dbg_default_trace("entering {}.",__func__); - struct stat stbuf; - (void) fi; + dbg_default_trace("entering {}.", __func__); + struct stat stbuf; // TODO memory leak? + (void)fi; std::memset(&stbuf, 0, sizeof(stbuf)); stbuf.st_ino = ino; - FCC_REQ(req)->fill_stbuf_by_ino(stbuf); + fcc(req)->fill_stbuf_by_ino(stbuf); fuse_reply_attr(req, &stbuf, 10000.0); - dbg_default_trace("leaving {}.",__func__); + dbg_default_trace("leaving {}.", __func__); } // borrowed from libfuse \a hello_ll.c struct dirbuf { - char *p; + char* p; size_t size; }; -static void dirbuf_add(fuse_req_t req, struct dirbuf *b, const char *name, fuse_ino_t ino) { +static void dirbuf_add(fuse_req_t req, struct dirbuf* b, const char* name, fuse_ino_t ino) { struct stat stbuf; size_t oldsize = b->size; b->size += fuse_add_direntry(req, nullptr, 0, name, nullptr, 0); - b->p = (char*)realloc(b->p,b->size); - std::memset(&stbuf,0,sizeof(stbuf)); + b->p = (char*)realloc(b->p, b->size); + std::memset(&stbuf, 0, sizeof(stbuf)); stbuf.st_ino = ino; - FCC_REQ(req)->fill_stbuf_by_ino(stbuf); - fuse_add_direntry(req, b->p+oldsize, b->size-oldsize, name, &stbuf, b->size); + fcc(req)->fill_stbuf_by_ino(stbuf); // additional effect + fuse_add_direntry(req, b->p + oldsize, b->size - oldsize, name, &stbuf, b->size); } -#define min(x, y) ((x) < (y) ? (x) : (y)) +static int reply_buf_limited(fuse_req_t req, const char* buf, size_t bufsize, + off_t off, size_t maxsize) { + if(static_cast(off) < bufsize) + return fuse_reply_buf(req, buf + off, std::min(bufsize - off, maxsize)); + else + return fuse_reply_buf(req, nullptr, 0); +} static void fs_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, struct fuse_file_info* fi) { - dbg_default_trace("entering {}.",__func__); + dbg_default_trace("entering {}.", __func__); struct dirbuf b; std::memset(&b, 0, sizeof(b)); dirbuf_add(req, &b, ".", 1); dirbuf_add(req, &b, "..", 1); - for(auto kv:FCC_REQ(req)->get_dir_entries(ino)) { + for(auto kv : fcc(req)->get_dir_entries(ino)) { dirbuf_add(req, &b, kv.first.c_str(), kv.second); } - if (static_cast(off) < b.size) { - fuse_reply_buf(req, b.p + off, min(b.size - off, size)); - } else { - fuse_reply_buf(req, NULL, 0); - } + reply_buf_limited(req, b.p, b.size, off, size); free(b.p); - dbg_default_trace("leaving {}.",__func__); + dbg_default_trace("leaving {}.", __func__); } static void fs_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { - dbg_default_trace("entering {}.",__func__); + dbg_default_trace("entering {}.", __func__); int err; - if ((fi->flags & O_ACCMODE) != O_RDONLY) { + if((fi->flags & O_ACCMODE) != O_RDONLY && !fcc(req)->is_writable(ino)) { + // allow write for key nodes fuse_reply_err(req, EACCES); - } else if ((err = FCC_REQ(req)->open_file(ino, fi)) != 0) { + } else if((err = fcc(req)->open_file(ino, fi)) != 0) { fuse_reply_err(req, err); } else { - dbg_default_debug("fi({:x})->fh={:x}",reinterpret_cast(fi),fi->fh); + dbg_default_debug("fi({:x})->fh={:x}", reinterpret_cast(fi), fi->fh); fuse_reply_open(req, fi); } - dbg_default_trace("leaving {}.",__func__); + dbg_default_trace("leaving {}.", __func__); } static void fs_read(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, struct fuse_file_info* fi) { dbg_default_trace("entering {}.", __func__); - FileBytes* pfb = reinterpret_cast(fi->fh); - if (static_cast(off) < pfb->size) { - fuse_reply_buf(req, reinterpret_cast(pfb->bytes+off), min(pfb->size - off, size)); + // TODO `file_bytes.bytes` might be freed in reply buf once fs_read ends. may need to go back to storing in `fi` + FileBytes file_bytes; + int res = fcc(req)->read_file(ino, size, off, fi, &file_bytes); + if(res == 0) { + reply_buf_limited(req, reinterpret_cast(file_bytes.bytes.data()), file_bytes.bytes.size(), off, size); } else { - fuse_reply_buf(req, nullptr, 0); + // TODO handle error + fuse_reply_err(req, 1); } + dbg_default_trace("leaving {}.", __func__); } static void fs_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { - dbg_default_trace("entering {}.",__func__); - FCC_REQ(req)->close_file(ino, fi); - fuse_reply_err(req,0); - dbg_default_trace("leaving {}.",__func__); + dbg_default_trace("entering {}.", __func__); + fcc(req)->close_file(ino, fi); + fuse_reply_err(req, 0); + dbg_default_trace("leaving {}.", __func__); +} + +static void fs_write(fuse_req_t req, fuse_ino_t ino, const char* buf, size_t size, off_t off, + struct fuse_file_info* fi) { + dbg_default_trace("entering {}.", __func__); + + // TODO write_buf instead?... use fi?? + ssize_t res = fcc(req)->write_file(ino, buf, size, off, fi); + if(res < 0) { + fuse_reply_err(req, -res); + } else { + fuse_reply_write(req, (size_t)res); + } + + dbg_default_trace("leaving {}.", __func__); +} + +static void fs_setattr(fuse_req_t req, fuse_ino_t ino, struct stat* attr, + int valid, struct fuse_file_info* fi) { + dbg_default_trace("setattr(ino={%u}, valid: {%o}).", ino, valid); + + // char procname[64]; + // struct lo_inode* inode = lo_inode(req, ino); + // int ifd = inode->fd; + int res; + + try { + if(!fi) { + errno = EACCES; + throw 1; + } + if(valid & FUSE_SET_ATTR_SIZE) { + dbg_default_trace("attr_size(length={})", (unsigned long)attr->st_size); + + if(!fcc(req)->is_writable(ino)) { + errno = EACCES; + throw 1; + } + res = fcc(req)->truncate(ino, attr->st_size, fi); + + if(res == -1) { + errno = EACCES; + throw 1; + } + } + // if(valid & FUSE_SET_ATTR_MODE) {} + // if(valid & (FUSE_SET_ATTR_UID | FUSE_SET_ATTR_GID)) {} + // if(valid & (FUSE_SET_ATTR_ATIME | FUSE_SET_ATTR_MTIME)) {} + return fs_getattr(req, ino, fi); + } catch(int& ex) { + int saverr = errno; + fuse_reply_err(req, saverr); + } +} + +static void fs_create(fuse_req_t req, fuse_ino_t parent, const char* name, + mode_t mode, struct fuse_file_info* fi) { + struct fuse_entry_param e; + int err; + + dbg_default_info("fs create: {}", name); + // TODO fi fields like direct_io and keep_cache + + err = fcc(req)->open_at(parent, name, mode, fi); + if(err) { + return (void)fuse_reply_err(req, err); + } + + err = do_lookup(req, parent, name, &e); + if(err) + fuse_reply_err(req, err); + else + fuse_reply_create(req, &e, fi); +} + +static void fs_mknod(fuse_req_t req, fuse_ino_t parent, const char* name, + mode_t mode, dev_t rdev) { + dbg_default_info("fs mknod: {}", name); + fuse_reply_err(req, 1); +} + +static void fs_mkdir(fuse_req_t req, fuse_ino_t parent, const char* name, + mode_t mode) { + dbg_default_info("fs mkdir: {}", name); + fuse_reply_err(req, 1); } static const struct fuse_lowlevel_ops fs_ops = { - .init = fs_init, - .destroy = fs_destroy, - .lookup = fs_lookup, - .forget = NULL, - .getattr = fs_getattr, - .setattr = NULL, - .readlink = NULL, - .mknod = NULL, - .mkdir = NULL, - .unlink = NULL, - .rmdir = NULL, - .symlink = NULL, - .rename = NULL, - .link = NULL, - .open = fs_open, - .read = fs_read, - .write = NULL, - .flush = NULL, - .release = fs_release, - .fsync = NULL, - .opendir = NULL, - .readdir = fs_readdir, + .init = fs_init, + .destroy = fs_destroy, + .lookup = fs_lookup, + .forget = nullptr, + .getattr = fs_getattr, + .setattr = fs_setattr, + .readlink = nullptr, + .mknod = fs_mknod, + .mkdir = fs_mkdir, + .unlink = nullptr, + .rmdir = nullptr, + .symlink = nullptr, + .rename = nullptr, + .link = nullptr, + .open = fs_open, + .read = fs_read, + .write = fs_write, + .release = fs_release, + .fsync = nullptr, + .opendir = nullptr, + .readdir = fs_readdir, + .create = fs_create, }; - /** * According to our experiment as well as recorded in * [this](https://www.cs.hmc.edu/~geoff/classes/hmc.cs135.201109/homework/fuse/fuse_doc.html) document as following: @@ -195,113 +318,115 @@ static const struct fuse_lowlevel_ops fs_ops = { * appear to work fine under the debugger. To avoid the problem, either (a) use absolute pathnames, or (b) record * your current working directory by calling get_current_dir_name before you invoke fuse_main, and then convert * relative pathnames into corresponding absolute ones. Obviously, (b) is the preferred approach. ", - * we need to prepare the configuration file path to make sure the \aServerClient is able to access the configuration + * we need to prepare the configuration file path to make sure the \a ServerClient is able to access the configuration * file. */ void prepare_derecho_conf_file() { - char cwd[4096]; -#pragma GCC diagnostic ignored "-Wunused-result" - getcwd(cwd,4096); -#pragma GCC diagnostic pop - sprintf(cwd+strlen(cwd), "/derecho.cfg"); - setenv("DERECHO_CONF_FILE", cwd, false); - dbg_default_debug("Using derecho config file:{}.", getenv("DERECHO_CONF_FILE")); -} - -void log_current_dir(bool foreground) { - char buff[FILENAME_MAX]; //create string buffer to hold path - auto working = GetCurrentDir( buff, FILENAME_MAX ); - if(working){ - std::string current_working_dir(buff); - - current_working_dir = (foreground)? current_working_dir + " (running in foreground)": current_working_dir + " (running in background)"; - auto derecho_conf_file = getenv("DERECHO_CONF_FILE"); - - std::ofstream out("/home/yy354/fuse_log.txt"); - out << current_working_dir; - out << " derecho conf file: [" <" << std::endl; + if(opts.show_help) { + std::cout << "usage: " << argv[0] << " [options] \n" + << std::endl; fuse_cmdline_help(); fuse_lowlevel_help(); ret = 0; throw 1; - } else if (opts.show_version) { + } else if(opts.show_version) { std::cout << "FUSE library version " << fuse_pkgversion() << std::endl; fuse_lowlevel_version(); ret = 0; throw 1; } - if (opts.mountpoint == nullptr) { - std::cout << "usage: " << argv[0] << " [options] " << std::endl; + if(opts.mountpoint == nullptr) { + std::cout << "usage: " << argv[0] << " [options] \n" + << " " << argv[0] << " --help" << std::endl; ret = 1; throw 1; } - fuse_daemonize(opts.foreground); // start session + // TODO fcc hangs forever when no server nodes running + // fcc also causes signal handlers to not register in fuse_set_signal_handlers + // problem: ServiceClientAPI::get_service_client() registers SIGINT and SIGTERM handlers + // (proof in logging??: [trace] Polling thread ending.) + // fuse_set_signal_handlers only sets when replacing SIG_DFL + FuseClientContextType fcc; + if(fuse_client_signals::store_old_signal_handlers() == -1) { + dbg_default_error("could not store old signal handlers"); + ret = 1; + throw 1; + } + + dbg_default_info("start session"); se = fuse_session_new(&args, &fs_ops, sizeof(fs_ops), &fcc); - if (se == nullptr) { + if(se == nullptr) { throw 1; } - if (fuse_set_signal_handlers(se) != 0) { + if(fuse_set_signal_handlers(se) != 0) { throw 2; } - if (fuse_session_mount(se, opts.mountpoint) != 0) { + if(fuse_session_mount(se, opts.mountpoint) != 0) { throw 3; } - //log_current_dir(opts.foreground); + fuse_daemonize(opts.foreground); - /* Block until ctrl+c or fuserount -u */ - if (opts.singlethread) { + /* Block until ctrl+c or fusermount -u */ + dbg_default_info("starting fuse client."); + if(opts.singlethread) { ret = fuse_session_loop(se); } else { - ret = fuse_session_loop_mt(se, opts.clone_fd); + std::cout << "Multi-threaded client not supported yet" << std::endl; + ret = 1; + // ret = fuse_session_loop_mt(se, opts.clone_fd); } - + + dbg_default_info("ending fuse."); fuse_session_unmount(se); - } - catch (int& ex) { + throw 3; + } catch(int& ex) { switch(ex) { - case 3: - fuse_remove_signal_handlers(se); - case 2: - fuse_session_destroy(se); - case 1: - free(opts.mountpoint); - fuse_opt_free_args(&args); + case 3: + fuse_remove_signal_handlers(se); + case 2: + fuse_session_destroy(se); + case 1: + if(fuse_client_signals::restore_old_signal_handlers() == -1) { + dbg_default_error("could not restore old signal handlers"); + ret = 1; + } + free(opts.mountpoint); + fuse_opt_free_args(&args); } } diff --git a/src/service/fuse/fuse_client_context.hpp b/src/service/fuse/fuse_client_context.hpp index e20f1c6c..29498d60 100644 --- a/src/service/fuse/fuse_client_context.hpp +++ b/src/service/fuse/fuse_client_context.hpp @@ -1,30 +1,37 @@ #pragma once -#include -#include -#include -#include -#include -#include + +#define FUSE_USE_VERSION 31 + +#include #include +#include #include +#include #include -#include + +#include +#include +#include +#include +#include +#include +#include +#include #include -#include "cascade/object_pool_metadata.hpp" +#include #include -#define GetCurrentDir getcwd namespace derecho { -namespace cascade{ +namespace cascade { using json = nlohmann::json; -#define FUSE_CLIENT_DEV_ID (0xCA7CADE) -#define FUSE_CLIENT_BLK_SIZE (4096) +#define FUSE_CLIENT_DEV_ID (0xCA7CADE) +#define FUSE_CLIENT_BLK_SIZE (4096) -#define META_FILE_NAME ".cascade" +#define META_FILE_NAME ".cascade" -#define TO_FOREVER (std::numeric_limits::max()) +#define TO_FOREVER (std::numeric_limits::max()) typedef enum { SITE = 0, @@ -41,20 +48,27 @@ typedef enum { class FileBytes { public: - size_t size; - uint8_t* bytes; - FileBytes():size(0),bytes(nullptr){} - FileBytes(size_t s):size(s) { - bytes = nullptr; - if (s > 0) { - bytes = (uint8_t*)malloc(s); - } - } - virtual ~FileBytes() { - if (bytes){ - free(bytes); - } - } + // TODO use Blob instead??? + std::vector bytes; + + // static FileBytes* from_fh(uint64_t fh) { + // return reinterpret_cast(fh); + // } + // public: + // size_t size; + // uint8_t* bytes; + // FileBytes() : size(0), bytes(nullptr) {} + // FileBytes(size_t s) : size(s) { + // bytes = nullptr; + // if(s > 0) { + // bytes = (uint8_t*)malloc(s); + // } + // } + // virtual ~FileBytes() { + // if(bytes) { + // free(bytes); + // } + // } }; class FuseClientINode { @@ -71,10 +85,10 @@ class FuseClientINode { * get directory entries. This is the default implementation. * Override it as required. */ - virtual std::map get_dir_entries() { - std::map ret_map; - for (auto& child: this->children) { - ret_map.emplace(child->display_name,reinterpret_cast(child.get())); + virtual std::map get_dir_entries() { + std::map ret_map; + for(auto& child : this->children) { + ret_map.emplace(child->display_name, reinterpret_cast(child.get())); } return ret_map; } @@ -83,52 +97,169 @@ class FuseClientINode { return sizeof(*this); } + virtual uint64_t open_at(const char* name, mode_t mode, struct fuse_file_info* fi) { + return 0; + } + virtual uint64_t read_file(FileBytes* fb) { - (void) fb; + (void)fb; return 0; } - virtual void initialize(){ + virtual uint64_t write_file(const char* buf, size_t size, off_t off, struct fuse_file_info* fi) { + (void)buf; + (void)size; + (void)off; + (void)fi; + + return -1; + } + + virtual uint64_t truncate(size_t size) { + return -1; + } + + virtual void initialize() { } - + // Helper function for get_dir_entries() and read_file() - void check_update(){ + void check_update() { struct timespec now; clock_gettime(CLOCK_REALTIME, &now); - if (now.tv_sec > (last_update_sec + update_interval)){ + // TODO why double if ??? + if(now.tv_sec > (last_update_sec + update_interval)) { clock_gettime(CLOCK_REALTIME, &now); - if (now.tv_sec > (last_update_sec + update_interval)){ - last_update_sec = now.tv_sec; + if(now.tv_sec > (last_update_sec + update_interval)) { + last_update_sec = now.tv_sec; update_contents(); } } } private: - // Helper functions for check_update() + // Helper functions for check_update() virtual void update_contents() { return; } }; +class FuseDataINode : public FuseClientINode { +public: + FileBytes cached_data; // TODO not necessary / bad idea since very expensive?? + persistent::version_t version; + uint64_t timestamp_us; + persistent::version_t previous_version; + // previous version by key, INVALID_VERSION for the first value of the key. + persistent::version_t previous_version_by_key; + + ServiceClientAPI& capi; + + FuseDataINode(ServiceClientAPI& capi) : cached_data(FileBytes()), capi(capi) { + } + + virtual uint64_t get_file_size() override { + check_update(); + return cached_data.bytes.size(); + } + + virtual uint64_t read_file(FileBytes* file_bytes) override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + + check_update(); + // TODO necessary copy?? + file_bytes->bytes = cached_data.bytes; + + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return 0; + } + + virtual uint64_t write_file(const char* buf, size_t size, off_t off, + struct fuse_file_info* fi) { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + check_update(); + + if(fi) { + std::stringstream ss; + ss << std::oct << fi->flags; + std::string flag_str = ss.str(); + + dbg_default_info("file flag: {}", flag_str); + if(fi->flags & O_LARGEFILE) { // TODO + dbg_default_info("large file"); + } + if(fi->flags & O_APPEND) { + dbg_default_info("appending"); + // TODO faster write ?? + } + if(fi->flags & O_TRUNC) { // TODO handled in open not here + dbg_default_info("truncating"); + } + } + + size_t next_size = std::max(cached_data.bytes.size(), off + size); + // if(!appending) { // TODO truncating happens elsewhere + // next_size = off + size; // remove extra old content + // } + dbg_default_info("writing. off: {}, buf_size: {}, next_size: {}.", off, size, next_size); + dbg_default_info("writing content: {}.", std::quoted(reinterpret_cast(buf))); + + FileBytes contents = cached_data; + contents.bytes.resize(next_size); + memcpy(contents.bytes.data() + off, buf, size); + // FileBytes next_bytes(next_size); + // memcpy(next_bytes.bytes, file_bytes->bytes, next_size); + // memcpy(next_bytes.bytes + off, buf, size); + + dbg_default_info("next file: {}.", std::quoted(reinterpret_cast(contents.bytes.data()))); + write_contents(contents.bytes.data(), next_size); + + update_contents(); // TODO race conditions ... ?? + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return 0; + } + + virtual uint64_t truncate(size_t size) { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + check_update(); + + dbg_default_info("truncating to {}.", size); + FileBytes contents = cached_data; + contents.bytes.resize(size); + write_contents(contents.bytes.data(), size); + + update_contents(); // TODO race conditions ... ?? + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return 0; + } + +private: + virtual void update_contents() { + return; + } + + virtual void write_contents(const uint8_t* content, size_t size) { + return; + } +}; template -struct TypeName { static const char *name; }; +struct TypeName { + static const char* name; +}; template -const char *TypeName::name = "unknown"; +const char* TypeName::name = "unknown"; template <> -const char *TypeName::name = "VolatileCascadeStoreWithStringKey"; +const char* TypeName::name = "VolatileCascadeStoreWithStringKey"; template <> -const char *TypeName::name = "PersistentCascadeStoreWithStringKey"; +const char* TypeName::name = "PersistentCascadeStoreWithStringKey"; template <> -const char *TypeName::name = "TriggerCascadeNoStoreWithStringKey"; +const char* TypeName::name = "TriggerCascadeNoStoreWithStringKey"; -#define CONF_LAYOUT "layout" - +#define CONF_LAYOUT "layout" template class SubgroupINode; @@ -163,25 +294,24 @@ template class CascadeTypeINode : public FuseClientINode { public: CascadeTypeINode() { - this->type = INodeType::CASCADE_TYPE; - this->display_name = std::string(TypeName::name); - this->parent = FUSE_ROOT_ID; + this->type = INodeType::CASCADE_TYPE; + this->display_name = std::string(TypeName::name); + this->parent = FUSE_ROOT_ID; } /** initialize */ void initialize(const json& group_layout, ServiceClientAPI& capi) { this->display_name = group_layout["type_alias"]; - uint32_t sidx=0; - for (auto subgroup_it:group_layout[CONF_LAYOUT]) { - - children.emplace_back(std::make_unique>(sidx,reinterpret_cast(this))); + uint32_t sidx = 0; + for(auto subgroup_it : group_layout[CONF_LAYOUT]) { + children.emplace_back(std::make_unique>(sidx, reinterpret_cast(this))); size_t num_shards = subgroup_it[MIN_NODES_BY_SHARD].size(); - for (uint32_t shidx = 0; shidx < num_shards; shidx ++) { + for(uint32_t shidx = 0; shidx < num_shards; shidx++) { this->children[sidx]->children.emplace_back( - std::make_unique>( - shidx,reinterpret_cast(this->children[sidx].get()),capi)); + std::make_unique>( + shidx, reinterpret_cast(this->children[sidx].get()), capi)); } - sidx ++; + sidx++; } } }; @@ -189,10 +319,9 @@ class CascadeTypeINode : public FuseClientINode { class RootMetaINode : public FuseClientINode { ServiceClientAPI& capi; std::string contents; - + public: - RootMetaINode (ServiceClientAPI& _capi) : - capi(_capi) { + RootMetaINode(ServiceClientAPI& capi) : capi(capi) { this->update_interval = 2; this->last_update_sec = 0; this->type = INodeType::META; @@ -200,24 +329,24 @@ class RootMetaINode : public FuseClientINode { } virtual uint64_t get_file_size() override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - check_update(); + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + check_update(); return contents.size(); } private: - virtual void update_contents () override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); + virtual void update_contents() override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); auto members = capi.get_members(); contents = "number of nodes in cascade service: " + std::to_string(members.size()) + ".\nnode IDs: "; - for (auto& nid : members) { + for(auto& nid : members) { contents += std::to_string(nid) + ","; } contents += "\n"; - - auto objectpools = capi.list_object_pools(false,true); + + auto objectpools = capi.list_object_pools(true); contents += "number of objectpool in cascade service: " + std::to_string(objectpools.size()) + ".\nObjectpool paths: "; - for (auto& objectpool_path : objectpools) { + for(auto& objectpool_path : objectpools) { contents += objectpool_path + ","; } contents += "\n"; @@ -225,8 +354,9 @@ class RootMetaINode : public FuseClientINode { virtual uint64_t read_file(FileBytes* file_bytes) override { this->check_update(); - file_bytes->size = contents.size(); - file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); + file_bytes->bytes.assign(contents.begin(), contents.end()); + // file_bytes->size = contents.size(); + // file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); return 0; } }; @@ -236,7 +366,7 @@ class SubgroupINode : public FuseClientINode { public: const uint32_t subgroup_index; - SubgroupINode (uint32_t sidx, fuse_ino_t pino) : subgroup_index(sidx) { + SubgroupINode(uint32_t sidx, fuse_ino_t pino) : subgroup_index(sidx) { this->type = INodeType::SUBGROUP; this->display_name = "subgroup-" + std::to_string(sidx); this->parent = pino; @@ -250,41 +380,40 @@ class ShardINode : public FuseClientINode { ServiceClientAPI& capi; std::map key_to_ino; - ShardINode (uint32_t shidx, fuse_ino_t pino, ServiceClientAPI& _capi) : - shard_index (shidx), capi(_capi) { + ShardINode(uint32_t shidx, fuse_ino_t pino, ServiceClientAPI& capi) : shard_index(shidx), capi(capi) { this->type = INodeType::SHARD; this->display_name = "shard-" + std::to_string(shidx); this->parent = pino; SubgroupINode* p_subgroup_inode = reinterpret_cast*>(pino); - this->children.emplace_back(std::make_unique>(shidx,p_subgroup_inode->subgroup_index,capi)); + this->children.emplace_back(std::make_unique>(shidx, p_subgroup_inode->subgroup_index, capi)); } // TODO: rethinking about the consistency - virtual std::map get_dir_entries() override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - //std::map ret_map; + virtual std::map get_dir_entries() override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + // std::map ret_map; /** we always retrieve the key list for a shard inode because the data is highly dynamic */ uint32_t subgroup_index = reinterpret_cast*>(this->parent)->subgroup_index; - auto result = capi.template list_keys(CURRENT_VERSION, true, subgroup_index, this->shard_index); - for (auto& reply_future:result.get()) { + auto result = capi.template list_keys(CURRENT_VERSION, true, subgroup_index, this->shard_index); + for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); std::unique_lock wlck(this->children_mutex); - for (auto& key: reply) { - if (key_to_ino.find(key) == key_to_ino.end()) { - this->children.emplace_back(std::make_unique>(key,reinterpret_cast(this),capi)); - // To solve the issue of '/' in display name , which will cause: reading directory '.': input/output error - // TODO: if there are better replacement of /, instead of - - key = std::string("key-") + key; - std::replace( key.begin(), key.end(), '/', '-'); + for(auto& key : reply) { + if(key_to_ino.find(key) == key_to_ino.end()) { + this->children.emplace_back(std::make_unique>(key, reinterpret_cast(this), capi)); + // To solve the issue of '/' in display name , which will cause: reading directory '.': input/output error + // TODO: if there are better replacement of /, instead of - + key = std::string("key-") + key; + std::replace(key.begin(), key.end(), '/', '-'); key_to_ino[key] = reinterpret_cast(this->children.back().get()); } } } - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); - return FuseClientINode::get_dir_entries(); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return FuseClientINode::get_dir_entries(); } }; - + template <> class ShardINode : public FuseClientINode { public: @@ -292,19 +421,18 @@ class ShardINode : public FuseClientINode { ServiceClientAPI& capi; std::map key_to_ino; - ShardINode (uint32_t shidx, fuse_ino_t pino, ServiceClientAPI& _capi) : - shard_index (shidx), capi(_capi) { + ShardINode(uint32_t shidx, fuse_ino_t pino, ServiceClientAPI& capi) : shard_index(shidx), capi(capi) { this->type = INodeType::SHARD; this->display_name = "shard-" + std::to_string(shidx); this->parent = pino; SubgroupINode* p_subgroup_inode = reinterpret_cast*>(pino); - this->children.emplace_back(std::make_unique>(shidx,p_subgroup_inode->subgroup_index,capi)); + this->children.emplace_back(std::make_unique>(shidx, p_subgroup_inode->subgroup_index, capi)); } - virtual std::map get_dir_entries() override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - std::map ret_map; - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); + virtual std::map get_dir_entries() override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + std::map ret_map; + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); return ret_map; } }; @@ -320,52 +448,51 @@ class ShardMetaINode : public FuseClientINode { /** * update the metadata. need write lock. */ - virtual void update_contents () override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - auto members = capi.template get_shard_members(subgroup_index,shard_index); + virtual void update_contents() override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + auto members = capi.template get_shard_members(subgroup_index, shard_index); contents = "number of nodes shard: " + std::to_string(members.size()) + ".\nnode IDs: "; - for (auto& nid : members) { + for(auto& nid : members) { contents += std::to_string(nid) + ","; } contents += "\n"; - auto policy = capi.template get_member_selection_policy(subgroup_index,shard_index); + auto policy = capi.template get_member_selection_policy(subgroup_index, shard_index); contents += "member selection policy:"; switch(std::get<0>(policy)) { - case FirstMember: - contents += "FirstMember\n"; - break; - case LastMember: - contents += "LastMember\n"; - break; - case Random: - contents += "Random\n"; - break; - case FixedRandom: - contents += "FixedRandom("; - contents += std::to_string(std::get<1>(policy)); - contents += ")\n"; - break; - case RoundRobin: - contents += "RoundRobin\n"; - break; - case UserSpecified: - contents += "UserSpecified("; - contents += std::to_string(std::get<1>(policy)); - contents += ")\n"; - break; - default: - contents += "Unknown\n"; - break; + case FirstMember: + contents += "FirstMember\n"; + break; + case LastMember: + contents += "LastMember\n"; + break; + case Random: + contents += "Random\n"; + break; + case FixedRandom: + contents += "FixedRandom("; + contents += std::to_string(std::get<1>(policy)); + contents += ")\n"; + break; + case RoundRobin: + contents += "RoundRobin\n"; + break; + case UserSpecified: + contents += "UserSpecified("; + contents += std::to_string(std::get<1>(policy)); + contents += ")\n"; + break; + default: + contents += "Unknown\n"; + break; } - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } + public: - ShardMetaINode (const uint32_t _shard_index, const uint32_t _subgroup_index, ServiceClientAPI& _capi) : - shard_index(_shard_index), - subgroup_index(_subgroup_index), - capi (_capi) { + ShardMetaINode(const uint32_t shard_index, const uint32_t subgroup_index, ServiceClientAPI& capi) + : shard_index(shard_index), subgroup_index(subgroup_index), capi(capi) { this->update_interval = 2; - this->last_update_sec = 0; + this->last_update_sec = 0; this->type = INodeType::META; this->display_name = META_FILE_NAME; } @@ -376,206 +503,201 @@ class ShardMetaINode : public FuseClientINode { } virtual uint64_t read_file(FileBytes* file_bytes) override { - check_update(); - file_bytes->size = contents.size(); - file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); - return 0; + check_update(); + file_bytes->bytes.assign(contents.begin(), contents.end()); + // file_bytes->size = contents.size(); + // file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); + return 0; } }; - template -class KeyINode : public FuseClientINode { +class KeyINode : public FuseDataINode { public: typename CascadeType::KeyType key; - std::unique_ptr file_bytes; - uint64_t file_size; - persistent::version_t version; - uint64_t timestamp_us; - persistent::version_t previous_version; - persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. - ServiceClientAPI& capi; - - KeyINode(typename CascadeType::KeyType& k, fuse_ino_t pino, ServiceClientAPI& _capi) : - key(k), - file_bytes(std::make_unique()), - capi(_capi){ + + KeyINode(typename CascadeType::KeyType& k, fuse_ino_t pino, ServiceClientAPI& capi) + : FuseDataINode(capi), key(k) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); - this->update_interval = 2; + this->update_interval = 2; this->last_update_sec = 0; this->type = INodeType::KEY; - if constexpr (std::is_same, char*>::value || - std::is_same, std::string>::value) { + if constexpr(std::is_same, char*>::value || std::is_same, std::string>::value) { this->display_name = std::string("key-") + k; - } else if constexpr (std::is_arithmetic>::value) { + } else if constexpr(std::is_arithmetic>::value) { this->display_name = std::string("key-") + std::to_string(k); } else { // KeyType is required to implement to_string() for types other than type string/arithmetic. this->display_name = std::string("key-") + key.to_string(); } // '/' in display name, will cause: reading directory '.': input/output error - std::replace( this->display_name.begin(), this->display_name.end(), '/', '\\'); - this->parent = pino; - dbg_default_trace("[{}]leaving {}.", gettid(), __func__); - } - - virtual uint64_t read_file(FileBytes* _file_bytes) override { - dbg_default_trace("[{}]entering {}.", gettid(), __func__); - check_update(); - _file_bytes->size = this->file_bytes.get()->size; - _file_bytes->bytes = static_cast(malloc(this->file_bytes.get()->size)); - memcpy(_file_bytes->bytes,this->file_bytes.get()->bytes, this->file_bytes.get()->size); + std::replace(this->display_name.begin(), this->display_name.end(), '/', '\\'); + this->parent = pino; dbg_default_trace("[{}]leaving {}.", gettid(), __func__); - return 0; } - virtual uint64_t get_file_size() override { - check_update(); - return this->file_bytes.get()->size; - } - - KeyINode(KeyINode&& fci){ + KeyINode(KeyINode&& fci) { this->type = std::move(fci.type); this->display_name = std::move(fci.display_name); this->parent = std::move(fci.parent); - this->key = std::move(fci.key ); + this->key = std::move(fci.key); this->capi = std::move(fci.capi); } - virtual ~KeyINode() { - dbg_default_info("[{}] entering {}.", gettid(), __func__); - file_bytes.reset(nullptr); - dbg_default_info("[{}] leaving {}.", gettid(), __func__); - } private: - virtual void update_contents () override{ - ShardINode *pino_shard = reinterpret_cast*>(this->parent); - SubgroupINode *pino_subgroup = reinterpret_cast*>(pino_shard->parent); + virtual void update_contents() override { + ShardINode* pino_shard = reinterpret_cast*>(this->parent); + SubgroupINode* pino_subgroup = reinterpret_cast*>(pino_shard->parent); auto result = capi.template get( - key,CURRENT_VERSION,true,pino_subgroup->subgroup_index,pino_shard->shard_index); - Blob blob; - for(auto& reply_future:result.get()){ - auto reply = reply_future.second.get(); - dbg_default_trace("------ KEY INODE reply {}.", reply); - if(std::is_base_of::value){ - ObjectWithStringKey *access_ptr = reinterpret_cast(&reply); - this->version = access_ptr->version; - this->timestamp_us = access_ptr->timestamp_us; - this->previous_version = access_ptr->previous_version; - this->previous_version_by_key = access_ptr->previous_version_by_key; - blob = access_ptr->blob; - }else if(std::is_base_of::value){ - ObjectWithUInt64Key *access_ptr = reinterpret_cast(&reply); - this->version = access_ptr->version; - this->timestamp_us = access_ptr->timestamp_us; - this->previous_version = access_ptr->previous_version; - this->previous_version_by_key = access_ptr->previous_version_by_key; - blob = access_ptr->blob; + key, CURRENT_VERSION, true, pino_subgroup->subgroup_index, pino_shard->shard_index); + Blob blob; + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + dbg_default_trace("------ KEY INODE reply {}.", reply); + if(std::is_base_of::value) { + ObjectWithStringKey* access_ptr = reinterpret_cast(&reply); + this->version = access_ptr->version; + this->timestamp_us = access_ptr->timestamp_us; + this->previous_version = access_ptr->previous_version; + this->previous_version_by_key = access_ptr->previous_version_by_key; + blob = access_ptr->blob; + } else if(std::is_base_of::value) { + ObjectWithUInt64Key* access_ptr = reinterpret_cast(&reply); + this->version = access_ptr->version; + this->timestamp_us = access_ptr->timestamp_us; + this->previous_version = access_ptr->previous_version; + this->previous_version_by_key = access_ptr->previous_version_by_key; + blob = access_ptr->blob; + } else { + // this->file_bytes.get()->size = mutils::bytes_size(reply); + // this->file_bytes.get()->bytes = static_cast(malloc(this->file_bytes.get()->size)); + // reply.to_bytes(this->file_bytes.get()->bytes); + this->cached_data.bytes.resize(mutils::bytes_size(reply)); + reply.to_bytes(this->cached_data.bytes.data()); + return; + } + // file_bytes.get()->size = blob.size; + // file_bytes.get()->bytes = static_cast(malloc(file_bytes.get()->size)); + // memcpy(file_bytes.get()->bytes, blob.bytes, file_bytes.get()->size); + this->cached_data.bytes.resize(blob.size); + memcpy(this->cached_data.bytes.data(), blob.bytes, blob.size); + return; + return; } - else{ - this->file_bytes.get()->size = mutils::bytes_size(reply); - this->file_bytes.get()->bytes = static_cast(malloc(this->file_bytes.get()->size)); - reply.to_bytes(this->file_bytes.get()->bytes); - return; - } - file_bytes.get()->size = blob.size; - file_bytes.get()->bytes = static_cast(malloc(file_bytes.get()->size)); - memcpy(file_bytes.get()->bytes, blob.bytes, file_bytes.get()->size); - return; + } + + virtual void write_contents(const uint8_t* content, size_t size) override { + ShardINode* pino_shard = reinterpret_cast*>(this->parent); + SubgroupINode* pino_subgroup = reinterpret_cast*>(pino_shard->parent); + + typename CascadeType::ObjectType obj; + obj.key = this->key; + // TODO + obj.previous_version = INVALID_VERSION; // this->previous_version; + obj.previous_version_by_key = INVALID_VERSION; // this->previous_version_by_key; + obj.blob = Blob(content, size, true); // TODO safe to emplace? + + dbg_default_debug("raw put"); + derecho::rpc::QueryResults> result + = capi.template put(obj, pino_subgroup->subgroup_index, pino_shard->shard_index); + + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + dbg_default_debug("node({}) replied with version:{},ts_us:{}", + reply_future.first, std::get<0>(reply), std::get<1>(reply)); } } }; - class ObjectPoolMetaINode : public FuseClientINode { private: std::string cur_pathname; /** objp_collection contains the all the objp with the same cur_pathname prefix - * i.e. if cur_path name is "/a", + * i.e. if cur_path name is "/a", * object pools "/a/b1", "/a/b2" share this level of ObjectPoolPathINode*/ std::vector objp_collection; - bool is_object_pool; - uint32_t subgroup_type_index; - uint32_t subgroup_index; + bool is_object_pool; + uint32_t subgroup_type_index; + uint32_t subgroup_index; sharding_policy_t sharding_policy; - bool deleted; + bool deleted; ServiceClientAPI& capi; std::string contents; - virtual void update_contents () override{ + virtual void update_contents() override { this->contents = "Current Directory Pathname: "; - this->contents += (cur_pathname=="") ? "objectPoolRoot" : cur_pathname; - this->contents += "\n"; - this->objp_collection.clear(); - this->contents += "contains the below object pools in its subdirs:\n"; - std::string objp_contents; - // Check the objectPools under cur_pathname directory - size_t cur_len = cur_pathname.length(); - for (std::string pathname : capi.list_object_pools(false,true)) { - if(cur_pathname.length() == 0 || (pathname.length() > cur_len - 1 && (pathname.substr(0,cur_len) == cur_pathname) )){ - if(pathname.length() == cur_len){ - this->objp_collection.emplace_back(pathname); - this->contents += " " + pathname + ",\n"; - this->is_object_pool = true; - this->objectPool_contents( objp_contents); - }else if(pathname[cur_len] == '/'){ - this->objp_collection.emplace_back(pathname); - this->contents += " " + pathname + ",\n"; + this->contents += (cur_pathname == "") ? "objectPoolRoot" : cur_pathname; + this->contents += "\n"; + this->objp_collection.clear(); + this->contents += "contains the below object pools in its subdirs:\n"; + std::string objp_contents; + // Check the objectPools under cur_pathname directory + size_t cur_len = cur_pathname.length(); + for(std::string pathname : capi.list_object_pools(true)) { + if(cur_pathname.length() == 0 || (pathname.length() > cur_len - 1 && (pathname.substr(0, cur_len) == cur_pathname))) { + if(pathname.length() == cur_len) { + this->objp_collection.emplace_back(pathname); + this->contents += " " + pathname + ",\n"; + this->is_object_pool = true; + this->objectPool_contents(objp_contents); + } else if(pathname[cur_len] == '/') { + this->objp_collection.emplace_back(pathname); + this->contents += " " + pathname + ",\n"; + } + } } - } - } - this->contents += objp_contents; + this->contents += objp_contents; } - + /** * Only fill the object pool contents when cur_pathname is an object pool */ - void objectPool_contents ( std::string& objp_contents) { + void objectPool_contents(std::string& objp_contents) { auto op_metadata = capi.find_object_pool(this->cur_pathname); - objp_contents += "Current Object Pool Pathname: "; - objp_contents += cur_pathname + "\n"; + objp_contents += "Current Object Pool Pathname: "; + objp_contents += cur_pathname + "\n"; this->deleted = op_metadata.deleted; - objp_contents += "- is deleted: "; - objp_contents += this->deleted? "true": "false"; - objp_contents += "\n"; + objp_contents += "- is deleted: "; + objp_contents += this->deleted ? "true" : "false"; + objp_contents += "\n"; this->subgroup_type_index = op_metadata.subgroup_type_index; - objp_contents += "- subgroup type index: "; - objp_contents += std::to_string(this->subgroup_type_index) + "\n"; + objp_contents += "- subgroup type index: "; + objp_contents += std::to_string(this->subgroup_type_index) + "\n"; this->subgroup_index = op_metadata.subgroup_index; - objp_contents += "- subgroup index: "; - objp_contents += std::to_string(this->subgroup_index) + "\n"; + objp_contents += "- subgroup index: "; + objp_contents += std::to_string(this->subgroup_index) + "\n"; auto policy = op_metadata.sharding_policy; objp_contents += "- sharding policy: "; switch(policy) { - case HASH: - objp_contents += "Hashing\n"; - break; - case RANGE: - objp_contents += "Range\n"; - break; - default: - objp_contents += "Unknown\n"; - break; + case HASH: + objp_contents += "Hashing\n"; + break; + case RANGE: + objp_contents += "Range\n"; + break; + default: + objp_contents += "Unknown\n"; + break; } } + public: - ObjectPoolMetaINode (const std::string _cur_pathname, ServiceClientAPI& _capi) : - cur_pathname(_cur_pathname), - capi(_capi) { + ObjectPoolMetaINode(const std::string cur_pathname, ServiceClientAPI& capi) + : cur_pathname(cur_pathname), capi(capi) { this->update_interval = 2; this->last_update_sec = 0; this->type = INodeType::META; this->display_name = META_FILE_NAME; } - void add_objp(std::string new_objp_pathname){ - for(auto &pathname : objp_collection){ - if(pathname == new_objp_pathname){ - return; - } - } - objp_collection.emplace_back(new_objp_pathname); - } + void add_objp(std::string new_objp_pathname) { + for(auto& pathname : objp_collection) { + if(pathname == new_objp_pathname) { + return; + } + } + objp_collection.emplace_back(new_objp_pathname); + } virtual uint64_t get_file_size() override { check_update(); @@ -583,13 +705,13 @@ class ObjectPoolMetaINode : public FuseClientINode { } virtual uint64_t read_file(FileBytes* file_bytes) override { - check_update(); - file_bytes->size = contents.size(); - file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); - return 0; + check_update(); + file_bytes->bytes.assign(contents.begin(), contents.end()); + // file_bytes->size = contents.size(); + // file_bytes->bytes = reinterpret_cast(strdup(contents.c_str())); + return 0; } }; - class ObjectPoolPathINode : public FuseClientINode { public: @@ -599,261 +721,273 @@ class ObjectPoolPathINode : public FuseClientINode { std::set key_children; std::set objp_children; - - ObjectPoolPathINode (fuse_ino_t pino, ServiceClientAPI& _capi): - capi(_capi){ - this->update_interval = 10; - this->last_update_sec = 0; + ObjectPoolPathINode(fuse_ino_t pino, ServiceClientAPI& capi) : capi(capi) { + this->update_interval = 10; + this->last_update_sec = 0; this->type = INodeType::OBJECTPOOL_PATH; this->parent = pino; - this->is_object_pool = false; - this->cur_pathname = ""; - this->children.emplace_back(std::make_unique(cur_pathname, capi)); + this->is_object_pool = false; + this->cur_pathname = ""; + this->children.emplace_back(std::make_unique(cur_pathname, capi)); } - ObjectPoolPathINode (std::string _cur_pathname,fuse_ino_t pino, ServiceClientAPI& _capi) : - capi(_capi), - cur_pathname(_cur_pathname){ + ObjectPoolPathINode(std::string cur_pathname, fuse_ino_t pino, ServiceClientAPI& capi) + : capi(capi), cur_pathname(cur_pathname) { this->update_interval = 10; this->last_update_sec = 10; this->type = INodeType::OBJECTPOOL_PATH; - std::size_t pos = cur_pathname.find_last_of('/'); + std::size_t pos = cur_pathname.find_last_of('/'); this->display_name = cur_pathname.substr(pos + 1); this->parent = pino; - this->is_object_pool = false; - //this->objp_collection.emplace_back(objp_pathname); - this->children.emplace_back(std::make_unique(cur_pathname, capi)); + this->is_object_pool = false; + // this->objp_collection.emplace_back(objp_pathname); + this->children.emplace_back(std::make_unique(cur_pathname, capi)); } - /** Helper function: + /** Helper function: * @object_pool_pathname: the object pool pathname to parse * @return: next level pathname * e.g. if cur_pathname is "/a", for object_pool_pathname "/a/b/c" this function returns "/a/b" */ - std::string get_next_level_pathname(std::string &object_pool_pathname){ - size_t cur_len = cur_pathname.length(); - std::string remain_pathname = object_pool_pathname.substr(cur_len); + std::string get_next_level_pathname(std::string& object_pool_pathname) { + size_t cur_len = cur_pathname.length(); + std::string remain_pathname = object_pool_pathname.substr(cur_len); auto start_pos = remain_pathname.find("/"); - if (start_pos == std::string::npos){ + if(start_pos == std::string::npos) { return ""; } - // get the next level of directory pathname. + // get the next level of directory pathname. auto end_pos = remain_pathname.substr(start_pos + 1).find("/"); std::string next_level_pathname; - if (end_pos == std::string::npos){ - next_level_pathname = cur_pathname + remain_pathname.substr(start_pos ); - }else{ - next_level_pathname = cur_pathname + remain_pathname.substr(start_pos , end_pos + 1); + if(end_pos == std::string::npos) { + next_level_pathname = cur_pathname + remain_pathname.substr(start_pos); + } else { + next_level_pathname = cur_pathname + remain_pathname.substr(start_pos, end_pos + 1); } - return next_level_pathname; - } + return next_level_pathname; + } - /** Construct the next level objectPoolINodes, starting from remain pathname - *e.g./a/b/c should have three layers: /a, /a/b, /a/b/c - *if this->cur_pathname is "/a", whose child INode's cur_pathname is "/a/b", whose child INode "/a/b/c" + /** Construct the next level objectPoolINodes, starting from remain pathname + *e.g./a/b/c should have three layers: /a, /a/b, /a/b/c + *if this->cur_pathname is "/a", whose child INode's cur_pathname is "/a/b", whose child INode "/a/b/c" */ - void construct_nextlevel_objectpool_path(std::string &object_pool_pathname){ + void construct_nextlevel_objectpool_path(std::string& object_pool_pathname) { // check path_name - std::string next_level_pathname = get_next_level_pathname(object_pool_pathname); - if(next_level_pathname.length() == 0){ - return; - } - // Check if this objectPoolPathInode with the same pathname exists - // case 1: If this level pathname directory exists - for(auto& inode : this->children){ - if (inode->type == INodeType::OBJECTPOOL_PATH){ - if(static_cast(inode.get())->cur_pathname == next_level_pathname){ - return; - } - } + std::string next_level_pathname = get_next_level_pathname(object_pool_pathname); + if(next_level_pathname.length() == 0) { + return; } - // case2: If this level ObjectPoolPathInode doesn't exists, create one + // Check if this objectPoolPathInode with the same pathname exists + // case 1: If this level pathname directory exists + for(auto& inode : this->children) { + if(inode->type == INodeType::OBJECTPOOL_PATH) { + if(static_cast(inode.get())->cur_pathname == next_level_pathname) { + return; + } + } + } + // case2: If this level ObjectPoolPathInode doesn't exists, create one // std::unique_lock wlck(this->children_mutex); - this->children.emplace_back(std::make_unique(next_level_pathname,reinterpret_cast(this),capi)); - objp_children.insert(cur_pathname); + this->children.emplace_back(std::make_unique(next_level_pathname, reinterpret_cast(this), capi)); + objp_children.insert(cur_pathname); } - - void update_objpINodes(){ + void update_objpINodes() { size_t cur_len = cur_pathname.length(); - std::vector reply = capi.list_object_pools(false,true); - std::vector valid_subdirs; + std::vector reply = capi.list_object_pools(true); + std::vector valid_subdirs; // Check if need to add new ObjectPoolPathINode to children inodes - for (std::string &object_pool : reply) { - if(object_pool.length() < cur_len){ - continue; - } - if(object_pool == cur_pathname){ - this->is_object_pool = true; - continue; - } - bool is_subdir = cur_pathname.length() == 0 || (object_pool.substr(0,cur_len) == cur_pathname && object_pool[cur_len] == '/') ; - if(!is_subdir){ - continue; - } - std::string next_level_pathname(get_next_level_pathname(object_pool)); - valid_subdirs.emplace_back(next_level_pathname); - if( objp_children.find(next_level_pathname) == objp_children.end() ){ - construct_nextlevel_objectpool_path(object_pool); - } + for(std::string& object_pool : reply) { + if(object_pool.length() < cur_len) { + continue; + } + if(object_pool == cur_pathname) { + this->is_object_pool = true; + continue; + } + bool is_subdir = cur_pathname.length() == 0 || (object_pool.substr(0, cur_len) == cur_pathname && object_pool[cur_len] == '/'); + if(!is_subdir) { + continue; + } + std::string next_level_pathname(get_next_level_pathname(object_pool)); + valid_subdirs.emplace_back(next_level_pathname); + if(objp_children.find(next_level_pathname) == objp_children.end()) { + construct_nextlevel_objectpool_path(object_pool); + } + } + // Check if need to remove existing ObjectPoolPathINode from children inodes + if(std::find(reply.begin(), reply.end(), this->cur_pathname) == reply.end()) { + this->is_object_pool = false; + } + auto it = this->children.begin(); + while(it != this->children.end()) { + std::string name = cur_pathname + "/" + (*it)->display_name; + if((*it)->type == INodeType::OBJECTPOOL_PATH && (std::find(valid_subdirs.begin(), valid_subdirs.end(), name) == valid_subdirs.end())) { + objp_children.erase(objp_children.find(name)); + it = this->children.erase(it); + } else { + ++it; + } } - // Check if need to remove existing ObjectPoolPathINode from children inodes - if(std::find(reply.begin(), reply.end(), this->cur_pathname) == reply.end()){ - this->is_object_pool = false; } + + void update_keyINodes() { + // case1. if object pool of cur_pathname donesn't exists anymore, remove all the keyINode from children + if(!this->is_object_pool) { + auto it = this->children.begin(); + while(it != this->children.end()) { + if((*it)->type == INodeType::KEY) { + it = this->children.erase(it); + } else { + ++it; + } + } + return; + } + // case2. if cur_pathname is a valid object pool, refetch keys in this object pool + persistent::version_t version = CURRENT_VERSION; + std::vector>>> future_result = capi.list_keys(version, true, cur_pathname); + std::vector reply = capi.wait_list_keys(future_result); + // Check if need to add new keyINode to children inodes + for(auto& key : reply) { + if(key_children.find(key) == key_children.end()) { + this->children.emplace_back(std::make_unique(key, reinterpret_cast(this), capi)); + key_children.insert(key); + } + } + // Check if need to remove existing keyINode from children inodes auto it = this->children.begin(); - while(it != this->children.end()){ - std::string name = cur_pathname + "/" + (*it)->display_name; - if ((*it)->type == INodeType::OBJECTPOOL_PATH && (std::find(valid_subdirs.begin(),valid_subdirs.end(), name ) == valid_subdirs.end())){ - objp_children.erase(objp_children.find(name)); - it = this->children.erase(it); - }else{ - ++it; - } - } - } - - void update_keyINodes(){ - // case1. if object pool of cur_pathname donesn't exists anymore, remove all the keyINode from children - if(!this->is_object_pool){ - auto it = this->children.begin(); - while(it != this->children.end()){ - if ((*it)->type == INodeType::KEY){ - it = this->children.erase(it); - }else{ - ++it; - } - } - return; - } - // case2. if cur_pathname is a valid object pool, refetch keys in this object pool - persistent::version_t version = CURRENT_VERSION; - std::vector>>> future_result = capi.list_keys(version, true, cur_pathname); - std::vector reply = capi.wait_list_keys(future_result); - // Check if need to add new keyINode to children inodes - for (auto& key: reply) { - if(key_children.find(key) == key_children.end() ){ - this->children.emplace_back(std::make_unique(key,reinterpret_cast(this),capi)); - key_children.insert(key); - } - } - // Check if need to remove existing keyINode from children inodes - auto it = this->children.begin(); - while(it != this->children.end()){ - std::string name = cur_pathname + "/" + (*it)->display_name; - if ((*it)->type == INodeType::KEY && (std::find(reply.begin(),reply.end(), name ) == reply.end())){ - key_children.erase(key_children.find(name)); - it = this->children.erase(it); - - }else{ - ++it; - } - } - } - - - virtual std::map get_dir_entries() override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - check_update(); - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); - return FuseClientINode::get_dir_entries(); + while(it != this->children.end()) { + std::string name = cur_pathname + "/" + (*it)->display_name; + if((*it)->type == INodeType::KEY && (std::find(reply.begin(), reply.end(), name) == reply.end())) { + key_children.erase(key_children.find(name)); + it = this->children.erase(it); + + } else { + ++it; + } + } + return; + } + + virtual std::map get_dir_entries() override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + check_update(); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return FuseClientINode::get_dir_entries(); } + + virtual uint64_t open_at(const char* name, mode_t mode, struct fuse_file_info* fi) { + dbg_default_info("object pool open at"); + update_contents(); + if(!this->is_object_pool) { + return 1; + } + + ObjectWithStringKey obj; + obj.key = cur_pathname + "/" + name; // TODO replace with std path + obj.previous_version = CURRENT_VERSION; + obj.previous_version_by_key = CURRENT_VERSION; + obj.blob = Blob(); + + dbg_default_debug("object pool put"); + derecho::rpc::QueryResults> result = capi.put(obj); + + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + dbg_default_debug("node({}) replied with version:{},ts_us:{}", + reply_future.first, std::get<0>(reply), std::get<1>(reply)); + } + + update_contents(); + return 0; + } + private: - virtual void update_contents () override{ - update_objpINodes(); - update_keyINodes(); - } + virtual void update_contents() override { + update_objpINodes(); + update_keyINodes(); + } }; - + class ObjectPoolRootINode : public ObjectPoolPathINode { public: - ObjectPoolRootINode (ServiceClientAPI& _capi, fuse_ino_t pino=FUSE_ROOT_ID) : - ObjectPoolPathINode(pino, _capi) { + ObjectPoolRootINode(ServiceClientAPI& capi, fuse_ino_t pino = FUSE_ROOT_ID) : ObjectPoolPathINode(pino, capi) { this->display_name = "ObjectPools"; this->parent = pino; } - /** this function constructs the whole object pool directories at the beginning - * for all the object pools stored in metadata service - */ - virtual std::map get_dir_entries() override { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - check_update(); - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); - return FuseClientINode::get_dir_entries(); + /** this function constructs the whole object pool directories at the beginning + * for all the object pools stored in metadata service + */ + virtual std::map get_dir_entries() override { + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + check_update(); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return FuseClientINode::get_dir_entries(); } + private: - virtual void update_contents () override{ - update_objpINodes(); - update_keyINodes(); - } + virtual void update_contents() override { + update_objpINodes(); + update_keyINodes(); + } }; - -class ObjectPoolKeyINode : public FuseClientINode{ +class ObjectPoolKeyINode : public FuseDataINode { public: std::string key; - std::unique_ptr file_bytes; - persistent::version_t version; - uint64_t timestamp_us; - persistent::version_t previous_version; - persistent::version_t previous_version_by_key; // previous version by key, INVALID_VERSION for the first value of the key. - ServiceClientAPI& capi; - ObjectPoolKeyINode(std::string k, fuse_ino_t pino, ServiceClientAPI& _capi) : - key(k), - file_bytes(std::make_unique()), - capi(_capi){ + ObjectPoolKeyINode(std::string k, fuse_ino_t pino, ServiceClientAPI& capi) + : FuseDataINode(capi), key(k) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); this->update_interval = 2; this->last_update_sec = 0; std::size_t pos = k.find_last_of('/'); this->display_name = k.substr(pos + 1); this->parent = pino; - this->type = INodeType::KEY; + this->type = INodeType::KEY; dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } - virtual uint64_t read_file(FileBytes* _file_bytes) override { - dbg_default_debug("-- READ FILE of key:[{}], [{}]entering {}.",this->key ,gettid(), __func__); - check_update(); - _file_bytes->size = this->file_bytes.get()->size; - _file_bytes->bytes = static_cast(malloc(this->file_bytes.get()->size)); - memcpy(_file_bytes->bytes,this->file_bytes.get()->bytes, this->file_bytes.get()->size); - dbg_default_debug("[{}]leaving {}.", gettid(), __func__); - return 0; +private: + virtual void update_contents() override { + dbg_default_debug("\n \n ----OBJP keyInode key is:[{}] - update content [{}] entering {}.", this->key, gettid(), __func__); + Blob blob; + auto result = capi.get(key, CURRENT_VERSION, true); + for(auto& reply_future : result.get()) { + auto reply = reply_future.second.get(); + ObjectWithStringKey* access_ptr = reinterpret_cast(&reply); + this->version = access_ptr->version; + this->timestamp_us = access_ptr->timestamp_us; + this->previous_version = access_ptr->previous_version; + this->previous_version_by_key = access_ptr->previous_version_by_key; + blob = access_ptr->blob; + // this->file_bytes.get()->size = blob.size; + // this->file_bytes.get()->bytes = static_cast(malloc(blob.size)); + // memcpy(this->file_bytes.get()->bytes, blob.bytes, blob.size); + this->cached_data.bytes.resize(blob.size); + memcpy(this->cached_data.bytes.data(), blob.bytes, blob.size); + return; + } + dbg_default_trace("\n \n ----OBJP keyInode update content [{}] leaving {}.", gettid(), __func__); } - virtual uint64_t get_file_size() override { - dbg_default_debug("----GET FILE SIZE key is [{}].", this->key); - check_update(); - return this->file_bytes.get()->size; - } + virtual void write_contents(const uint8_t* content, size_t size) override { + ObjectWithStringKey obj; + obj.key = this->key; + obj.previous_version = INVALID_VERSION; + obj.previous_version_by_key = INVALID_VERSION; + obj.blob = Blob(content, size, true); // TODO safe to emplace? - virtual ~ObjectPoolKeyINode() { - dbg_default_info("[{}] entering {}.", gettid(), __func__); - file_bytes.reset(nullptr); - dbg_default_info("[{}] leaving {}.", gettid(), __func__); - } -private: - virtual void update_contents () override{ - dbg_default_debug("\n \n ----OBJP keyInode key is:[{}] - update content [{}] entering {}.", this->key ,gettid(), __func__); - Blob blob; - auto result = capi.get(key,CURRENT_VERSION,true); - for (auto& reply_future:result.get()) { + dbg_default_debug("object pool put"); + derecho::rpc::QueryResults> result = capi.put(obj); + + for(auto& reply_future : result.get()) { auto reply = reply_future.second.get(); - ObjectWithStringKey *access_ptr = reinterpret_cast(&reply); - this->version = access_ptr->version; - this->timestamp_us = access_ptr->timestamp_us; - this->previous_version = access_ptr->previous_version; - this->previous_version_by_key = access_ptr->previous_version_by_key; - blob = access_ptr->blob; - this->file_bytes.get()->size = blob.size; - this->file_bytes.get()->bytes = static_cast(malloc(blob.size)); - memcpy(this->file_bytes.get()->bytes, blob.bytes, blob.size); - return; + dbg_default_debug("node({}) replied with version:{},ts_us:{}", + reply_future.first, std::get<0>(reply), std::get<1>(reply)); } - dbg_default_trace("\n \n ----OBJP keyInode update content [{}] leaving {}.", gettid(), __func__); } }; @@ -861,32 +995,29 @@ class MetadataServiceRootINode : public FuseClientINode { ServiceClientAPI& capi; public: - MetadataServiceRootINode(ServiceClientAPI& _capi) : - capi(_capi) { + MetadataServiceRootINode(ServiceClientAPI& capi) : capi(capi) { this->type = INodeType::METADATA_SERVICE; this->display_name = "MetadataService"; this->parent = FUSE_ROOT_ID; } void initialize() { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); + dbg_default_trace("[{}]entering {}.", gettid(), __func__); children.emplace_back(std::make_unique(capi, reinterpret_cast(this))); this->children.back().get()->initialize(); children.emplace_back(std::make_unique(capi, reinterpret_cast(this))); this->children.back().get()->initialize(); } - }; - /** - * TODO: add DLL FUSE support - */ +/** + * TODO: add DLL FUSE support + */ class DataPathLogicRootINode : public FuseClientINode { ServiceClientAPI& capi; public: - DataPathLogicRootINode (ServiceClientAPI& _capi, fuse_ino_t pino) : - capi(_capi) { + DataPathLogicRootINode(ServiceClientAPI& capi, fuse_ino_t pino) : capi(capi) { this->type = INodeType::DATAPATH_LOGIC; this->display_name = "DataPathLogic"; this->parent = FUSE_ROOT_ID; @@ -895,28 +1026,27 @@ class DataPathLogicRootINode : public FuseClientINode { /** initialize */ void initialize() { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); + dbg_default_trace("[{}]entering {}.", gettid(), __func__); } }; - - /** - * TODO: add DLL FUSE support - */ +/** + * TODO: add DLL FUSE support + */ template class DLLINode : public FuseClientINode { public: - std::string file_name; // DLL id? + std::string file_name; // DLL id? ServiceClientAPI& capi; - DLLINode(std::string& _filename, fuse_ino_t pino, ServiceClientAPI& _capi) : - file_name(_filename), capi(_capi) { + DLLINode(std::string& file_name, fuse_ino_t pino, ServiceClientAPI& capi) + : file_name(file_name), capi(capi) { dbg_default_trace("[{}]entering {}.", gettid(), __func__); this->type = INodeType::DLL; - this->display_name = std::string("dllfile") + _filename; + this->display_name = std::string("dllfile") + file_name; this->parent = pino; dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } - + virtual uint64_t read_file(FileBytes* file_bytes) override { dbg_default_trace("[{}]entering {}.", gettid(), __func__); dbg_default_trace("[{}]leaving {}.", gettid(), __func__); @@ -930,11 +1060,11 @@ class DLLINode : public FuseClientINode { return fsize; } - DLLINode(DLLINode&& fci){ + DLLINode(DLLINode&& fci) { this->type = std::move(fci.type); this->display_name = std::move(fci.display_name); this->parent = std::move(fci.parent); - this->file_name = std::move(fci.file_name ); + this->file_name = std::move(fci.file_name); this->capi = std::move(fci.capi); } @@ -944,14 +1074,16 @@ class DLLINode : public FuseClientINode { } }; - /** - * The fuse filesystem context for fuse_client. This context will be used as 'userdata' on starting a fuse session. + * The fuse filesystem context for fuse_client. + * This context will be used as 'userdata' on starting a fuse session. */ template class FuseClientContext { - template using _CascadeTypeINode = CascadeTypeINode; + template + using _CascadeTypeINode = CascadeTypeINode; + private: /** initialization flag */ std::atomic is_initialized; @@ -963,7 +1095,7 @@ class FuseClientContext { ServiceClientAPI& capi; /** The inodes are stored in \a inodes. */ - mutils::KindMap<_CascadeTypeINode,CascadeTypes...> inodes; + mutils::KindMap<_CascadeTypeINode, CascadeTypes...> inodes; /** Metadata */ RootMetaINode metadata_inode; @@ -976,12 +1108,12 @@ class FuseClientContext { /** fill inodes */ void populate_inodes(const json& group_layout) { - if (!group_layout.is_array()) { + if(!group_layout.is_array()) { dbg_default_error("JSON group layout is invalid (array expected): {}.", group_layout.get()); throw std::runtime_error("JSON group layout is invalid."); } // populate from the second layout, since the first one is for metadata service - do_populate_inodes(group_layout,1); + do_populate_inodes(group_layout, 1); } template @@ -991,41 +1123,40 @@ class FuseClientContext { template void do_populate_inodes(const json& group_layout, int type_idx) { this->do_populate_inodes(group_layout, type_idx); - this->do_populate_inodes(group_layout, type_idx+1); + this->do_populate_inodes(group_layout, type_idx + 1); } public: - FuseClientContext() : - capi(ServiceClientAPI::get_service_client()), - metadata_inode(capi), - objectpool_inode(capi), - admin_metadata_inode(capi){} + FuseClientContext() : capi(ServiceClientAPI::get_service_client()), + metadata_inode(capi), + objectpool_inode(capi), + admin_metadata_inode(capi) {} /** initialize */ void initialize(const json& group_layout) { - dbg_default_trace("[{}]entering {} .", gettid(), __func__); + dbg_default_trace("[{}]entering {}.", gettid(), __func__); populate_inodes(group_layout); this->admin_metadata_inode.initialize(); - clock_gettime(CLOCK_REALTIME,&this->init_timestamp); + clock_gettime(CLOCK_REALTIME, &this->init_timestamp); this->is_initialized.store(true); dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } /** read directory entries by ino */ - std::map get_dir_entries(fuse_ino_t ino) { + std::map get_dir_entries(fuse_ino_t ino) { dbg_default_trace("[{}]entering {} with ino ={:x}.", gettid(), __func__, ino); std::map ret_map; - if (ino == FUSE_ROOT_ID) { + if(ino == FUSE_ROOT_ID) { this->inodes.for_each( - [&ret_map](auto k,auto& v){ - // CAUTION: only works up to 64bit virtual address CPU architectures. - ret_map.emplace(v.display_name,reinterpret_cast(&v)); - }); - ret_map.emplace(metadata_inode.display_name,reinterpret_cast(&this->metadata_inode)); - ret_map.emplace(objectpool_inode.display_name,reinterpret_cast(&this->objectpool_inode)); - ret_map.emplace(admin_metadata_inode.display_name,reinterpret_cast(&this->admin_metadata_inode)); + [&ret_map](auto k, auto& v) { + // CAUTION: only works up to 64bit virtual address CPU architectures. + ret_map.emplace(v.display_name, reinterpret_cast(&v)); + }); + ret_map.emplace(metadata_inode.display_name, reinterpret_cast(&this->metadata_inode)); + ret_map.emplace(objectpool_inode.display_name, reinterpret_cast(&this->objectpool_inode)); + ret_map.emplace(admin_metadata_inode.display_name, reinterpret_cast(&this->admin_metadata_inode)); } else { FuseClientINode* pfci = reinterpret_cast(ino); - ret_map = pfci->get_dir_entries(); // RVO + ret_map = pfci->get_dir_entries(); // RVO } dbg_default_trace(" [{}]leaving {}.", gettid(), __func__); return ret_map; @@ -1037,107 +1168,169 @@ class FuseClientContext { * */ double fill_stbuf_by_ino(struct stat& stbuf) { - dbg_default_trace("[{}]entering {}.",gettid(),__func__); - double timeout_sec = 1.0; //TO_FOREVER; + dbg_default_trace("[{}]entering {}.", gettid(), __func__); + double timeout_sec = 1.0; // TO_FOREVER; // 1 - common attributes stbuf.st_dev = FUSE_CLIENT_DEV_ID; - stbuf.st_nlink = 1; + stbuf.st_nlink = 1; // TODO set property stbuf.st_uid = getuid(); stbuf.st_gid = getgid(); - stbuf.st_atim = this->init_timestamp; + stbuf.st_atim = this->init_timestamp; // TODO set properly stbuf.st_mtim = this->init_timestamp; stbuf.st_ctim = this->init_timestamp; // 2 - special attributes - if (stbuf.st_ino == FUSE_ROOT_ID) { + if(stbuf.st_ino == FUSE_ROOT_ID) { stbuf.st_mode = S_IFDIR | 0755; stbuf.st_size = FUSE_CLIENT_BLK_SIZE; stbuf.st_blocks = 1; stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; } else { FuseClientINode* pfci = reinterpret_cast(stbuf.st_ino); - switch (pfci->type) { - case INodeType::SITE: - // TODO: - break; - case INodeType::CASCADE_TYPE: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::METADATA_SERVICE: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::SUBGROUP: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::SHARD: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::KEY: - stbuf.st_mode = S_IFREG| 0444; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::META: - stbuf.st_mode = S_IFREG| 0444; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::OBJECTPOOL_PATH: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::DATAPATH_LOGIC: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - case INodeType::DLL: - stbuf.st_mode = S_IFDIR | 0755; - stbuf.st_size = pfci->get_file_size(); - stbuf.st_blocks = (stbuf.st_size+FUSE_CLIENT_BLK_SIZE-1)/FUSE_CLIENT_BLK_SIZE; - stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; - break; - default: - ; + switch(pfci->type) { + case INodeType::SITE: + // TODO: + break; + case INodeType::CASCADE_TYPE: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::METADATA_SERVICE: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::SUBGROUP: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::SHARD: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::KEY: + stbuf.st_mode = S_IFREG | 0744; // TODO set 0444 for unimplemented + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::META: + stbuf.st_mode = S_IFREG | 0444; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::OBJECTPOOL_PATH: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::DATAPATH_LOGIC: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + case INodeType::DLL: + stbuf.st_mode = S_IFDIR | 0755; + stbuf.st_size = pfci->get_file_size(); + stbuf.st_blocks = (stbuf.st_size + FUSE_CLIENT_BLK_SIZE - 1) / FUSE_CLIENT_BLK_SIZE; + stbuf.st_blksize = FUSE_CLIENT_BLK_SIZE; + break; + default:; } } - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); return timeout_sec; } + // TODO docs + bool is_writable(fuse_ino_t ino) { + FuseClientINode* pfci = reinterpret_cast(ino); + // TODO only allow for object pool?? (or not) + return pfci->type == INodeType::KEY; + } + + int open_at(fuse_ino_t parent, const char* name, mode_t mode, struct fuse_file_info* fi) { + FuseClientINode* pfci = reinterpret_cast(parent); + + return pfci->open_at(name, mode, fi); + } + + int read_file(fuse_ino_t ino, size_t size, off_t off, struct fuse_file_info* fi, FileBytes* file_bytes) { + FuseClientINode* pfci = reinterpret_cast(ino); + + size_t result = pfci->read_file(file_bytes); // TODO only read necessary? + + return result; // wrong number container size + } + + int write_file(fuse_ino_t ino, const char* buf, size_t size, off_t off, struct fuse_file_info* fi) { + dbg_default_trace("[{}]entering {} with ino={:x}.", gettid(), __func__, ino); + + if(!is_writable(ino)) { + return -EACCES; + } + + FuseClientINode* pfci = reinterpret_cast(ino); + pfci->write_file(buf, size, off, fi); + // TODO errors?? + ssize_t result = size; + + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); + return result; + } + + int truncate(fuse_ino_t ino, size_t size, struct fuse_file_info* fi) { + // TODO use errno ? + FuseClientINode* pfci = reinterpret_cast(ino); + + if(!is_writable(ino)) { + return -1; + } + + return pfci->truncate(size); + } + /** * open a file. * @param ino inode * @param fi file structure shared among processes opening this file. * @return error code. 0 for success. */ - int open_file(fuse_ino_t ino, struct fuse_file_info *fi) { + int open_file(fuse_ino_t ino, struct fuse_file_info* fi) { dbg_default_trace("[{}]entering {} with ino={:x}.", gettid(), __func__, ino); FuseClientINode* pfci = reinterpret_cast(ino); - if (pfci->type != INodeType::KEY && - pfci->type != INodeType::META) { + + if(fi->flags & O_CREAT) { + dbg_default_info("open: create flag"); + // TODO get parent inode and create + } + // TODO handle truncate + + if(fi->flags & O_TRUNC) { + dbg_default_info("open: trunc flag"); + pfci->truncate(0); // TODO cache (do finally on close file) + } + if(fi->flags & O_LARGEFILE) { + dbg_default_info("open: large file flag"); // TODO !! + } + + if(pfci->type != INodeType::KEY && pfci->type != INodeType::META) { return EISDIR; } - FileBytes* fb = new FileBytes(); - pfci->read_file(fb); - fi->fh = reinterpret_cast(fb); - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); + // TODO cleaner to not set fh to something non file handle like ?? + // FileBytes* fb = new FileBytes(); + // pfci->read_file(fb); + // fi->fh = reinterpret_cast(fb); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); return 0; } @@ -1147,16 +1340,17 @@ class FuseClientContext { * @param fi file structure shared among processes opening this file. * @return error code. 0 for success. */ - int close_file(fuse_ino_t ino, struct fuse_file_info *fi) { + int close_file(fuse_ino_t ino, struct fuse_file_info* fi) { dbg_default_trace("[{}]entering {} with ino={:x}.", gettid(), __func__, ino); - void* pfb = reinterpret_cast(fi->fh); - if (pfb!=nullptr) { - delete static_cast(pfb); - } + // TODO no longer need to delete? + // void* pfb = reinterpret_cast(fi->fh); + // if(pfb != nullptr) { + // delete static_cast(pfb); + // } return 0; - dbg_default_trace("[{}]leaving {}.",gettid(),__func__); + dbg_default_trace("[{}]leaving {}.", gettid(), __func__); } }; -} // namespace cascade -} // namespace derecho +} // namespace cascade +} // namespace derecho diff --git a/src/service/fuse/fuse_client_hl.cpp b/src/service/fuse/fuse_client_hl.cpp new file mode 100644 index 00000000..8fc95341 --- /dev/null +++ b/src/service/fuse/fuse_client_hl.cpp @@ -0,0 +1,515 @@ +#define HAVE_SETXATTR + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "fcc_hl.hpp" +#include "fuse_client_signals.hpp" +#include +#ifdef HAVE_SETXATTR +#include +#endif + +using namespace derecho::cascade; + +// TODO segfault debugging +// stat on invalid path + +struct cli_options { + const char* client_dir; + int update_interval; + int by_version; +}; + +static cli_options options = { + .client_dir = nullptr, + .update_interval = 15, + .by_version = 0, +}; + +#define OPTION(t, p) \ + { t, offsetof(struct cli_options, p), 1 } + +static const struct fuse_opt option_spec[] = { + OPTION("--client=%s", client_dir), + OPTION("--update-interval=%d", update_interval), + OPTION("--by_version", by_version), + FUSE_OPT_END}; + +static void show_help(const char* progname) { + printf("usage: %s [options] \n\n", progname); + printf(" --update-interval= Update-rate of file system contents (default: 15)\n" + " --client= Client directory\n" + " --by_version Snapshot by version number rather than timestamp in microseconds\n" + "\n"); +} + +// --- + +static FuseClientContext* fcc() { + return static_cast(fuse_get_context()->private_data); +} + +static void* cascade_fs_init(struct fuse_conn_info* conn, + struct fuse_config* cfg) { + // TODO why read conf_layout_json_layout? + // TODO don't like no control over derecho config + + return new FuseClientContext(options.update_interval, options.by_version); +} + +static void cascade_fs_destroy(void* private_data) { + delete static_cast(private_data); +} + +static int cascade_fs_getattr(const char* path, struct stat* stbuf, + struct fuse_file_info* fi) { + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + memset(stbuf, 0, sizeof(struct stat)); + return fcc()->get_stat(node, stbuf); +} + +// TODO :( invalid pointer dumped?? somehow cascade replys needs to be stored in a variable before +// result.get() ... maybe it somehow dangles? +static int cascade_fs_readdir(const char* path, void* buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info* fi, + enum fuse_readdir_flags flags) { + // TODO offsets for very large reads + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + // TODO fill stbuf properly? (check for get_stat errors) + // struct stat stbuf; + // memset(&stbuf, 0, sizeof(struct stat)); + + // FSTree::get_stat(node, &stbuf); + filler(buf, ".", nullptr, 0, (fuse_fill_dir_flags)0); + // TODO parent of root ??? + // FSTree::get_stat(node->parent, &stbuf); + filler(buf, "..", nullptr, 0, (fuse_fill_dir_flags)0); + for(const auto& [k, v] : node->children) { + if(filler(buf, k.c_str(), nullptr, 0, (fuse_fill_dir_flags)0)) { + break; + } + // FSTree::get_stat(v, &stbuf); + } + + return 0; +} + +static int cascade_fs_open(const char* path, struct fuse_file_info* fi) { + // TODO check O_ACCMODE, also check if dir ?? + auto node = fcc()->get(path); + if(fi->flags & O_CREAT && node == nullptr) { + node = fcc()->add_op_key(path); + if(node == nullptr) { + return -ENOTSUP; + } + } + if(node == nullptr) { + return -ENOENT; + } + if(node->data.flag & DIR_FLAG) { + return -ENOTSUP; + } + if(fi->flags & O_TRUNC) { + node->data.bytes.clear(); + } + + fi->fh = reinterpret_cast(node); + return 0; +} + +static int cascade_fs_create(const char* path, mode_t mode, + struct fuse_file_info* fi) { + // TODO check mode ? currently ignores + return cascade_fs_open(path, fi); +} + +static int cascade_fs_read(const char* path, char* buf, size_t size, off_t offset, + struct fuse_file_info* fi) { + // auto node = reinterpret_cast(fi->fh); + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + if(node->data.flag & DIR_FLAG) { // TODO diff error + return -EACCES; + } + auto& bytes = node->data.bytes; + + size_t len = bytes.size(); + + if((size_t)offset < len) { + if(offset + size > len) { + size = len - offset; + } + auto p = reinterpret_cast(bytes.data()); + memcpy(buf, p + offset, size); + } else { + size = 0; + } + + return size; +} + +static int cascade_fs_write(const char* path, const char* buf, size_t size, + off_t offset, struct fuse_file_info* fi) { + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + if(node->data.flag & DIR_FLAG || !node->data.writeable) { // TODO diff error + return -ENOTSUP; + } + auto& bytes = node->data.bytes; + + bytes.resize(std::max(bytes.size(), offset + size)); // TODO -ENOMEM + // TODO potentially undefined? + memcpy(bytes.data() + offset, buf, size); + + return size; +} + +/* +static int cascade_fs_flush(const char* path, struct fuse_file_info* fi) { + // TODO impl? (release is only called once at end while flush is called many times) + // There is no guarantee that flush will ever be called at all + return -ENOSYS; +} +*/ + +static int cascade_fs_release(const char* path, struct fuse_file_info* fi) { + auto node = fcc()->get(path); + if((fi->flags & O_ACCMODE) == O_RDONLY) { + return 0; + } + if(node == nullptr) { + return -ENOENT; + } + if(node->data.flag & DIR_FLAG || !node->data.writeable) { + return -ENOTSUP; + } + return fcc()->put_to_capi(node); +} + +static int cascade_fs_mkdir(const char* path, mode_t mode) { + if(fcc()->get(path)) { + return -EEXIST; + } + auto op_root = fcc()->nearest_object_pool_root(path); + if(op_root == nullptr) { + return fcc()->add_snapshot(path) ? 0 : -EACCES; + } + auto res = fcc()->add_op_key_dir(path); + if(res) { + fcc()->local_latest_dirs.insert(path); + // TODO err if nullptr + } + + return 0; +} + +static int cascade_fs_unlink(const char* path) { + // return -ENOTSUP; + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + if(node->data.flag & DIR_FLAG) { + return -EISDIR; + } + // TODO check open + + // remove + node->data.bytes.clear(); + + // auto result = capi.remove(key); + // node->parent->children.erase(node->label); + // delete node; + + return 0; +} + +static int cascade_fs_rmdir(const char* path) { + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + if(!(node->data.flag & DIR_FLAG)) { + return -ENOTDIR; + } + if((node->data.flag & KEY_DIR) == 0) { + return -EACCES; + } + if(!node->children.empty()) { + return -ENOTEMPTY; + } + + // remove + fcc()->local_latest_dirs.erase(path); + node->parent->children.erase(node->label); + delete node; + + return 0; +} + +static int cascade_fs_truncate(const char* path, off_t size, + struct fuse_file_info* fi) { + auto node = fcc()->get(path); + if(node == nullptr) { + return -ENOENT; + } + if(node->data.flag & DIR_FLAG || !node->data.writeable) { + return -EINVAL; + } + node->data.bytes.resize(size, 0); + return fcc()->put_to_capi(node); +} + +int set_buffer(char* dest, size_t size, const char* src, size_t len = 0) { + if(len == 0) { + len = strlen(src); + } + if(size == 0) { + return len; + } else if(size < len) { + return -ERANGE; + } + memcpy(dest, src, len); + return len; +} + +static int cascade_fs_chmod(const char* path, mode_t mode, struct fuse_file_info* fi) { + // TODO implement or -ENOTSUP + return 0; +} + +static int cascade_fs_chown(const char* path, uid_t uid, gid_t gid, struct fuse_file_info* fi) { + return -ENOTSUP; +} + +static int cascade_fs_utimens(const char*, const struct timespec tv[2], struct fuse_file_info* fi) { + // TODO implement or -ENOTSUP + return 0; +} + +#ifdef HAVE_SETXATTR + +/* +static int cascade_fs_setxattr(const char* path, const char* name, const char* value, + size_t size, int flags) { + if(flags & XATTR_CREATE) { + return -EPERM; + } + dbg_default_info("{}, {}, {}", path, name, value); + // TODO save into node data + if(strcmp(path, fcc()->ROOT) == 0) { + if(!fcc()->latest && strcmp(name, "user.cascade.version") == 0) { + fcc()->ver = strtoll(value, nullptr, 0); + // TODO + fcc()->update_object_pools(); + return 0; + } else if(strcmp(name, "user.cascade.latest") == 0) { + fcc()->latest = strcmp(value, "1") == 0; + fcc()->update_object_pools(); + return 0; + } + } + return -EPERM; +} +*/ + +static int cascade_fs_getxattr(const char* path, const char* name, char* value, + size_t size) { + // TODO need to first apt get-install attr + if(strcmp(path, fcc()->ROOT.c_str()) == 0) { + if(strcmp(name, "user.cascade.largest_known_version") == 0) { + std::string v = std::to_string(fcc()->max_ver); + return set_buffer(value, size, v.c_str()); + } + } + return -ENODATA; +} + +static int cascade_fs_listxattr(const char* path, char* list, size_t size) { + // TODO lesson learned :(. returned 0 instead of length. check over all return types + if(strcmp(path, fcc()->ROOT.c_str()) == 0) { + // ^ 1hr+ bug + const char names[] = "user.cascade.largest_known_version" + "\0"; + // "user.cascade.version" + // "\0" + // "user.cascade.latest" + // "\0"; + // very weird behavior with \0 termination. strings and determining sizes did not work well + // ^ 2hr+ bug + return set_buffer(list, size, names, sizeof(names) / sizeof(names[0]) - 1); + } + const char empty[] = ""; + return set_buffer(list, size, empty, 0); +} + +#endif + +// TODO create_object_pool, remove_object_pool, get_object_pool, op_remove +// op_get size. maybe no need to cache data? +// TODO version/#/... (mount) +// maybe use attr + +static const struct fuse_operations cascade_fs_oper = { + .getattr = cascade_fs_getattr, + // truncate, utimens, chown in place of setattr + .mkdir = cascade_fs_mkdir, + .unlink = cascade_fs_unlink, + .rmdir = cascade_fs_rmdir, + .chmod = cascade_fs_chmod, + .chown = cascade_fs_chown, + .truncate = cascade_fs_truncate, + .open = cascade_fs_open, + .read = cascade_fs_read, + .write = cascade_fs_write, + // .flush = cascade_fs_flush, + .release = cascade_fs_release, +#ifdef HAVE_SETXATTR + // .setxattr = cascade_fs_setxattr, + .getxattr = cascade_fs_getxattr, + .listxattr = cascade_fs_listxattr, +#endif + .readdir = cascade_fs_readdir, + .init = cascade_fs_init, + .destroy = cascade_fs_destroy, + .create = cascade_fs_create, + .utimens = cascade_fs_utimens}; + +bool prepare_derecho_conf_file(const char* config_dir) { + // TODO for some reason needs to already be in correct directory ??? + // maybe derecho_conf_file envvar not correct? + // std::filesystem::path p(config_dir); + // const std::string conf_file = (p / "derecho.cfg").u8string(); + + // std::filesystem::current_path(config_dir); + const std::string conf_file = (std::filesystem::current_path() / "derecho.cfg").u8string(); + + struct stat buffer; + if(stat(conf_file.c_str(), &buffer) != 0) { + return false; + } + + const char* const derecho_conf_file = "DERECHO_CONF_FILE"; + setenv(derecho_conf_file, conf_file.c_str(), false); + dbg_default_info("Using derecho config file: {}.", getenv(derecho_conf_file)); + return true; +} + +int main(int argc, char* argv[]) { + options.client_dir = strdup("."); + + struct fuse_args args = FUSE_ARGS_INIT(argc, argv); + struct fuse* fuse = nullptr; + struct fuse_cmdline_opts opts; + struct fuse_loop_config config; + int res = 0; + + if(fuse_opt_parse(&args, &options, option_spec, nullptr) == -1) { + return 1; + } + + if(fuse_parse_cmdline(&args, &opts) != 0) { + return 1; + } + + try { + if(opts.show_version) { + printf("FUSE library version %s\n", fuse_pkgversion()); + fuse_lowlevel_version(); + res = 0; + throw 1; + } else if(opts.show_help) { + show_help(argv[0]); + fuse_cmdline_help(); + // fuse_lib_help(&args); + res = 0; + throw 1; + } else if(!opts.mountpoint) { + fprintf(stderr, "error: no mountpoint specified\n"); + res = 1; + throw 1; + } else if(!opts.singlethread) { + fprintf(stderr, "error: multi-threaded client not supported\n"); + res = 1; + throw 1; + } else if(!prepare_derecho_conf_file(options.client_dir)) { + fprintf(stderr, "error: invalid client directory\n" + "(dir needs derecho.cfg if DERECHO_CONF_FILE envvar is not set)\n"); + res = 1; + throw 1; + } + + try { + fuse = fuse_new(&args, &cascade_fs_oper, sizeof(cascade_fs_oper), nullptr); + if(fuse == nullptr) { + res = 1; + throw 1; + } + + if(fuse_mount(fuse, opts.mountpoint) != 0) { + res = 1; + throw 2; + } + + if(fuse_daemonize(opts.foreground) != 0) { + res = 1; + throw 3; + } + + struct fuse_session* se = fuse_get_session(fuse); + if(fuse_set_signal_handlers(se) != 0) { + res = 1; + throw 3; + } + if(opts.singlethread) { + res = fuse_loop(fuse); + } else { + config.clone_fd = opts.clone_fd; + config.max_idle_threads = opts.max_idle_threads; + res = fuse_loop_mt(fuse, &config); + } + if(res) { + res = 1; + } + + fuse_remove_signal_handlers(se); + } catch(derecho::derecho_exception& ex) { + std::cerr << "Exception:" << ex.what() << std::endl; + // TODO errors are not caught :( like cannot bind to socket + res = 1; + } + + throw 3; + } catch(int& ex) { + switch(ex) { + case 3: + fuse_unmount(fuse); + case 2: + fuse_destroy(fuse); + case 1: + free(opts.mountpoint); + fuse_opt_free_args(&args); + } + } + + return res; +} diff --git a/src/service/fuse/fuse_client_signals.cpp b/src/service/fuse/fuse_client_signals.cpp new file mode 100644 index 00000000..5f68fd44 --- /dev/null +++ b/src/service/fuse/fuse_client_signals.cpp @@ -0,0 +1,49 @@ +#include "fuse_client_signals.hpp" + +#include +#include +#include + +using handler_t = void (*)(int); +static std::map old_signal_handlers; +static const int signals[] = {SIGHUP, SIGINT, SIGTERM, SIGPIPE}; + +static int set_one_signal_handler(int sig, int remove) { + if(remove) { + struct sigaction old_sa; + if(sigaction(sig, NULL, &old_sa) == -1) { + return -1; + } + old_signal_handlers[sig] = old_sa.sa_handler; + } + + struct sigaction sa; + memset(&sa, 0, sizeof(struct sigaction)); + bool contains = old_signal_handlers.find(sig) == old_signal_handlers.end(); + sa.sa_handler = remove || !contains ? SIG_DFL : old_signal_handlers[sig]; + sigemptyset(&(sa.sa_mask)); + sa.sa_flags = 0; + if(sigaction(sig, &sa, NULL) == -1) { + return -1; + } + + return 0; +} + +int fuse_client_signals::store_old_signal_handlers() { + for(int sig : signals) { + if(set_one_signal_handler(sig, true) == -1) { + return -1; + } + } + return 0; +} + +int fuse_client_signals::restore_old_signal_handlers() { + for(int sig : signals) { + if(set_one_signal_handler(sig, false) == -1) { + return -1; + } + } + return 0; +} diff --git a/src/service/fuse/fuse_client_signals.hpp b/src/service/fuse/fuse_client_signals.hpp new file mode 100644 index 00000000..366588eb --- /dev/null +++ b/src/service/fuse/fuse_client_signals.hpp @@ -0,0 +1,7 @@ +#pragma once + +namespace fuse_client_signals { +int store_old_signal_handlers(); + +int restore_old_signal_handlers(); +} // namespace fuse_client_signals diff --git a/src/service/fuse/path_tree.hpp b/src/service/fuse/path_tree.hpp new file mode 100644 index 00000000..fc9c4592 --- /dev/null +++ b/src/service/fuse/path_tree.hpp @@ -0,0 +1,159 @@ +#include +#include +namespace fs = std::filesystem; + +template +struct PathTree { + std::string label; + T data; + + PathTree* parent; + std::unordered_map*> children; + + PathTree(std::string label, T data, PathTree* parent) + : label(label), data(data), parent(parent) {} + PathTree(std::string label, T data) : PathTree(label, data, nullptr) {} + + ~PathTree() { + for(auto& [k, v] : children) { + delete v; + } + } + + std::vector entries() const { + std::vector res; + res.reserve(children.size()); + for(const auto& [k, _] : children) { + res.push_back(k); + } + return res; + } + + std::string absolute_path() const { + std::vector parts; + for(const PathTree* node = this; node != nullptr; node = node->parent) { + parts.push_back(node->label); + } + std::string res; + for(auto it = parts.rbegin(); it != parts.rend(); ++it) { + if(res.compare("") != 0 && res.compare("/") != 0) { + res += "/"; + } + res += *it; + } + + return res; + } + + void print(int depth = 0, std::ostream& stream = std::cout, + int pad = 0) const { + for(int i = 0; i < pad - 1; ++i) { + stream << " "; + } + if(pad) { + stream << "|-"; + } + stream << label << "\n"; + if(children.empty() || depth == 0) { + return; + } + for(const auto& [_, v] : children) { + v->print(depth - 1, stream, pad + 1); + } + if(pad == 0) { + stream << "\n\n"; + } + } + + // returns nullptr on fail or if location already exists + PathTree* set(const fs::path& path, T intermediate, T data) { + if(path.empty()) { + return nullptr; + } + auto it = path.begin(); + if(*it != label) { + return nullptr; + } + bool created_new = false; + PathTree* cur = this; + for(++it; it != path.end(); ++it) { + if(!cur->children.count(*it)) { + created_new = true; + PathTree* next = new PathTree(*it, intermediate, cur); + cur->children.insert({*it, next}); + cur = next; + } else { + cur = cur->children.at(*it); + } + } + if(!created_new) { + return nullptr; + } + cur->data = data; + return cur; + } + + // returns nullptr on fail or location does not exist + PathTree* get(const fs::path& path) { + if(path.empty()) { + return nullptr; + } + auto it = path.begin(); + if(*it != label) { + return nullptr; + } + PathTree* cur = this; + for(++it; it != path.end(); ++it) { + if(!cur->children.count(*it)) { + return nullptr; + } + cur = cur->children.at(*it); + } + return cur; + } + + PathTree* get_while_valid(const fs::path& path) { + if(path.empty()) { + return nullptr; + } + auto it = path.begin(); + if(*it != label) { + return nullptr; + } + PathTree* cur = this; + for(++it; it != path.end(); ++it) { + if(!cur->children.count(*it)) { + return cur; + } + cur = cur->children.at(*it); + } + return cur; + } + + PathTree* extract(const fs::path& path) { + // TODO use?? + PathTree* cur = get(path); + if(cur == nullptr || cur->parent == nullptr) { + return nullptr; + } + PathTree* par = cur->parent; + par->children.erase(cur->label); + + cur->parent = nullptr; + return cur; + } + + // deletes node and replaces with replacement. updating parent + // requires same label + static bool replace(PathTree* node, PathTree* replacement) { + if(node == nullptr || replacement == nullptr || node->label != replacement->label || node->parent == nullptr) { + return false; + } + std::cout << "replacing " << node->absolute_path() << std::endl; + PathTree* par = node->parent; + replacement->parent = par; + par->children[node->label] = replacement; + delete node; + return true; + } +}; diff --git a/src/service/fuse/testfuse.py b/src/service/fuse/testfuse.py index d1f0860a..d957b45b 100644 --- a/src/service/fuse/testfuse.py +++ b/src/service/fuse/testfuse.py @@ -1,176 +1,279 @@ #!/usr/bin/env python3 -from derecho.cascade.client import ServiceClientAPI -import subprocess +import errno +import filecmp +import getopt import multiprocessing -import threading import os -import sys,getopt -import stat +import queue import shutil -import filecmp +import stat +import subprocess +import sys import tempfile +import threading import time -import errno -import sys -import queue -from tempfile import NamedTemporaryFile from contextlib import contextmanager -from util import (wait_for_mount, compare_dirs, umount, cleanup, base_cmdline, - safe_sleep, basename, test_printcap, - fuse_proto, powerset) from os.path import join as pjoin +from tempfile import NamedTemporaryFile + +from derecho.cascade.client import ServiceClientAPI +from util import ( + base_cmdline, + basename, + cleanup, + compare_dirs, + fuse_proto, + powerset, + safe_sleep, + test_printcap, + umount, + wait_for_mount, +) TEST_FILE = __file__ -with open(TEST_FILE, 'rb') as fh: +with open(TEST_FILE, "rb") as fh: TEST_DATA = fh.read() + def name_generator(__ctr=[0]): __ctr[0] += 1 - return 'testfile_%d' % __ctr[0] + return "testfile_%d" % __ctr[0] + options = [] -if sys.platform == 'linux': - options.append('clone_fd') +if sys.platform == "linux": + options.append("clone_fd") def invoke_directly(mnt_dir, name, options): - cmdline = base_cmdline + [ pjoin(basename, 'example', name), - '-f', mnt_dir, '-o', ','.join(options) ] - if name == 'cascade_fuse_client': + cmdline = base_cmdline + [ + pjoin(basename, "example", name), + "-f", + mnt_dir, + "-o", + ",".join(options), + ] + if name == "cascade_fuse_client": # supports single-threading only - cmdline.append('-s') + cmdline.append("-s") return cmdline def readdir_inode(dir): - cmd = base_cmdline + [ pjoin(basename, 'test', 'readdir_inode'), dir ] - with subprocess.Popen(cmd, stdout=subprocess.PIPE, - universal_newlines=True) as proc: + cmd = base_cmdline + [pjoin(basename, "test", "readdir_inode"), dir] + with subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) as proc: lines = proc.communicate()[0].splitlines() lines.sort() return lines -def error_msg( run_id, testcase, output, expected): +def error_msg(run_id, testcase, output, expected): msg = "[" + run_id + "]" + testcase + "(output--expected):" - msg += "(" + ','.join(output) - msg += " -- " + ','.join(expected) + ")" + msg += "(" + ",".join(output) + msg += " -- " + ",".join(expected) + ")" return msg - -put_keys = ['key0', '/a/a1/key0', '/b/b1/key1', '/c/c1/c2/key2'] -put_values = ['00', '00000000', '11111111', '22222222'] -vcss_expected_keys = [".cascade","key-key0", "key-\\a\\a1\\key0"] -pcss_expected_keys = [".cascade","key-\\b\\b1\\key1","key-\\c\\c1\\c2\\key2"] +put_keys = ["key0", "/a/a1/key0", "/b/b1/key1", "/c/c1/c2/key2"] +put_values = ["00", "00000000", "11111111", "22222222"] + +vcss_expected_keys = [".cascade", "key-key0", "key-\\a\\a1\\key0"] +pcss_expected_keys = [".cascade", "key-\\b\\b1\\key1", "key-\\c\\c1\\c2\\key2"] objp_expected_second_level = [".cascade", "a", "b", "c"] -objp_expected_third_level = [[], [".cascade", "a1"], [".cascade", "b1"], [".cascade", "c1"]] -objp_expected_keys = [[], [".cascade", "key0"], [".cascade", "key1"], [".cascade", "c2"]] +objp_expected_third_level = [ + [], + [".cascade", "a1"], + [".cascade", "b1"], + [".cascade", "c1"], +] +objp_expected_keys = [ + [], + [".cascade", "key0"], + [".cascade", "key1"], + [".cascade", "c2"], +] objp_expected_keys2 = [".cascade", "key2"] - def initial_setup(): capi = ServiceClientAPI() print("----------- CASCADE INITIAL SETUP -----------") - capi.put(put_keys[0], bytes(put_values[0],'utf-8'), subgroup_type='VolatileCascadeStoreWithStringKey', subgroup_index=0, shard_index=0) - capi.create_object_pool('/a/a1', 'VolatileCascadeStoreWithStringKey', 0) - capi.create_object_pool("/b/b1",'PersistentCascadeStoreWithStringKey', 0) - capi.create_object_pool("/c/c1/c2",'PersistentCascadeStoreWithStringKey', 0) + capi.put( + put_keys[0], + bytes(put_values[0], "utf-8"), + subgroup_type="VolatileCascadeStoreWithStringKey", + subgroup_index=0, + shard_index=0, + ) + capi.create_object_pool("/a/a1", "VolatileCascadeStoreWithStringKey", 0) + capi.create_object_pool("/b/b1", "PersistentCascadeStoreWithStringKey", 0) + capi.create_object_pool("/c/c1/c2", "PersistentCascadeStoreWithStringKey", 0) time.sleep(5) - - capi.put(put_keys[1], bytes(put_values[1],'utf-8'),blocking=True) - capi.put(put_keys[2],bytes(put_values[2],'utf-8'),blocking=True) - res = capi.put(put_keys[3], bytes(put_values[3],'utf-8'),blocking=True) + + capi.put(put_keys[1], bytes(put_values[1], "utf-8"), blocking=True) + capi.put(put_keys[2], bytes(put_values[2], "utf-8"), blocking=True) + res = capi.put(put_keys[3], bytes(put_values[3], "utf-8"), blocking=True) if res: odict = res.get_result() # print("------- THRID put result: " + f"{str(odict)}") - + # Test 1. check the directory for Cascade subgroups def tst_sg_dirs(mnt_dir, run_id=""): # First-level(subgroup_type) check mnt_first_level = os.listdir(mnt_dir) - expected_first_level = ['.cascade', 'MetadataService', 'PersistentCascadeStoreWithStringKey', 'VolatileCascadeStoreWithStringKey', 'TriggerCascadeNoStoreWithStringKey','ObjectPools' ] - error_m = error_msg(run_id, "first level", mnt_first_level, expected_first_level) + expected_first_level = [ + ".cascade", + "MetadataService", + "PersistentCascadeStoreWithStringKey", + "VolatileCascadeStoreWithStringKey", + "TriggerCascadeNoStoreWithStringKey", + "ObjectPools", + ] + error_m = error_msg(run_id, "first level", mnt_first_level, expected_first_level) assert compare_dirs(mnt_first_level, expected_first_level), error_m sg_expected_second_level = ["subgroup-0"] sg_expected_third_level = ["shard-0"] # Second-level(subgroup_index) check for subgroup_type in expected_first_level: - if(subgroup_type=='.cascade' or subgroup_type=='MetadataService' or subgroup_type=='ObjectPools'): + if ( + subgroup_type == ".cascade" + or subgroup_type == "MetadataService" + or subgroup_type == "ObjectPools" + ): continue mnt_second_level = os.listdir(mnt_dir + "/" + subgroup_type) assert compare_dirs(mnt_second_level, sg_expected_second_level) # Third-level(shard) check - mnt_third_level = os.listdir(mnt_dir + "/" + subgroup_type + "/" + sg_expected_second_level[0]) + mnt_third_level = os.listdir( + mnt_dir + "/" + subgroup_type + "/" + sg_expected_second_level[0] + ) assert compare_dirs(mnt_third_level, sg_expected_third_level) # Forth-level(key) check - if(subgroup_type == 'PersistentCascadeStoreWithStringKey'): - mnt_forth_level = os.listdir(mnt_dir + "/" + subgroup_type + "/" + sg_expected_second_level[0] + "/" + sg_expected_third_level[0]) - error_m = error_msg(run_id, "PCSS forthlevel", mnt_forth_level, pcss_expected_keys) + if subgroup_type == "PersistentCascadeStoreWithStringKey": + mnt_forth_level = os.listdir( + mnt_dir + + "/" + + subgroup_type + + "/" + + sg_expected_second_level[0] + + "/" + + sg_expected_third_level[0] + ) + error_m = error_msg( + run_id, "PCSS forthlevel", mnt_forth_level, pcss_expected_keys + ) assert compare_dirs(mnt_forth_level, pcss_expected_keys), error_m - if(subgroup_type == 'VolatileCascadeStoreWithStringKey'): - mnt_forth_level = os.listdir(mnt_dir + "/" + subgroup_type + "/" + sg_expected_second_level[0] + "/" + sg_expected_third_level[0]) - assert compare_dirs(mnt_forth_level, vcss_expected_keys) - print("["+ run_id +"]"+ "--- Test1 SubgroupTypes directories pass! ---") + if subgroup_type == "VolatileCascadeStoreWithStringKey": + mnt_forth_level = os.listdir( + mnt_dir + + "/" + + subgroup_type + + "/" + + sg_expected_second_level[0] + + "/" + + sg_expected_third_level[0] + ) + assert compare_dirs(mnt_forth_level, vcss_expected_keys) + print("[" + run_id + "]" + "--- Test1 SubgroupTypes directories pass! ---") + - - # Test 2. check the directory for Cascade object_pools +# Test 2. check the directory for Cascade object_pools def tst_objp_dirs(mnt_dir, run_id=""): mnt_objp_root = os.listdir(mnt_dir + "/" + "ObjectPools") - error_m = error_msg(run_id, "object pool test", mnt_objp_root, objp_expected_second_level) + error_m = error_msg( + run_id, "object pool test", mnt_objp_root, objp_expected_second_level + ) assert compare_dirs(mnt_objp_root, objp_expected_second_level), error_m - for i in range(1,4): - mnt_objp_dir = os.listdir(mnt_dir + "/" + "ObjectPools" + "/" + objp_expected_second_level[i]) - error_m = error_msg(run_id, f"object pool test{str(i)}", mnt_objp_dir, objp_expected_third_level[i]) + for i in range(1, 4): + mnt_objp_dir = os.listdir( + mnt_dir + "/" + "ObjectPools" + "/" + objp_expected_second_level[i] + ) + error_m = error_msg( + run_id, + f"object pool test{str(i)}", + mnt_objp_dir, + objp_expected_third_level[i], + ) assert compare_dirs(mnt_objp_dir, objp_expected_third_level[i]) - mnt_objp_key = os.listdir(mnt_dir + "/" + "ObjectPools" + "/" + objp_expected_second_level[i] + "/" + objp_expected_third_level[i][1]) + mnt_objp_key = os.listdir( + mnt_dir + + "/" + + "ObjectPools" + + "/" + + objp_expected_second_level[i] + + "/" + + objp_expected_third_level[i][1] + ) assert compare_dirs(mnt_objp_key, objp_expected_keys[i]) - if(i == 3): - mnt_objp_subdir = os.listdir(mnt_dir + "/" + "ObjectPools" + "/" + objp_expected_second_level[i] + "/" + objp_expected_third_level[i][1] + "/" + objp_expected_keys[i][1]) + if i == 3: + mnt_objp_subdir = os.listdir( + mnt_dir + + "/" + + "ObjectPools" + + "/" + + objp_expected_second_level[i] + + "/" + + objp_expected_third_level[i][1] + + "/" + + objp_expected_keys[i][1] + ) assert compare_dirs(mnt_objp_subdir, objp_expected_keys2) print("[" + run_id + "]" + "--- Test2 ObjectPools directories pass! ---") - + # Test 3. check the open and read content of files def tst_content(mnt_dir, run_id=""): # 1. subgroup content test subgroup_type = "/VolatileCascadeStoreWithStringKey/" - for i in range(1,2): - sg_mnt_key = mnt_dir + subgroup_type + "subgroup-0/shard-0/" +vcss_expected_keys[i+1] - with open(sg_mnt_key, 'r') as fh: + for i in range(1, 2): + sg_mnt_key = ( + mnt_dir + subgroup_type + "subgroup-0/shard-0/" + vcss_expected_keys[i + 1] + ) + with open(sg_mnt_key, "r") as fh: output = fh.read() - error_m = error_msg(run_id, f"sg content{sg_mnt_key}", output, put_values[i]) + error_m = error_msg( + run_id, f"sg content{sg_mnt_key}", output, put_values[i] + ) assert output == put_values[i], error_m subgroup_type = "/PersistentCascadeStoreWithStringKey/" - for i in range(2,4): - sg_mnt_key = mnt_dir + subgroup_type + "subgroup-0/shard-0/" +pcss_expected_keys[i-1] - with open(sg_mnt_key, 'r') as fh: + for i in range(2, 4): + sg_mnt_key = ( + mnt_dir + subgroup_type + "subgroup-0/shard-0/" + pcss_expected_keys[i - 1] + ) + with open(sg_mnt_key, "r") as fh: output = fh.read() - error_m = error_msg(run_id, f"sg content{sg_mnt_key}", output, put_values[i]) + error_m = error_msg( + run_id, f"sg content{sg_mnt_key}", output, put_values[i] + ) assert output == put_values[i], error_m # 2. Objectpool content test object_pool_dir = "/ObjectPools" - for i in range(1,4): + for i in range(1, 4): objp_mnt_key = mnt_dir + object_pool_dir + put_keys[i] - with open(objp_mnt_key, 'r') as fh: + with open(objp_mnt_key, "r") as fh: output = fh.read() - error_m = error_msg(run_id, f"objp content{objp_mnt_key}", output, put_values[i]) + error_m = error_msg( + run_id, f"objp content{objp_mnt_key}", output, put_values[i] + ) assert output == put_values[i], error_m - print("["+ run_id +"]" + "--- Test3 read content pass! ---") + print("[" + run_id + "]" + "--- Test3 read content pass! ---") + # def tst_attr(mnt_dir): - + + def cascade_fuse_mount(mnt_dir): # 1. create mounting piont subprocess.Popen(["mkdir", mnt_dir]) - cmd_input = base_cmdline + [ pjoin(basename, "cascade_fuse_client"), mnt_dir, "-f"] + cmd_input = base_cmdline + [pjoin(basename, "cascade_fuse_client"), mnt_dir, "-f"] # 2. run cascade_fuse_client - mount_process = subprocess.Popen(cmd_input, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + mount_process = subprocess.Popen( + cmd_input, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) # try: wait_for_mount(mount_process, mnt_dir) # except: @@ -179,18 +282,19 @@ def cascade_fuse_mount(mnt_dir): # else: # umount(mnt_dir) + def cascade_fuse_umount(mnt_dir): # 1. umount the mounting process at mnt_dir umount(mnt_dir) # 2. delete the mounted directory - subprocess.Popen(["rm", "-r",mnt_dir]) + subprocess.Popen(["rm", "-r", mnt_dir]) print(" ~~~~~~~~~~~~~ FINISHED cascade_fuse_client unmount ~~~~~~~~~~~\n") def fuse_test_cases(mnt_dir, run_id): - tst_sg_dirs(mnt_dir,run_id) - tst_objp_dirs(mnt_dir,run_id) - tst_content(mnt_dir,run_id) + tst_sg_dirs(mnt_dir, run_id) + tst_objp_dirs(mnt_dir, run_id) + tst_content(mnt_dir, run_id) # tst_attr(mnt) @@ -199,33 +303,41 @@ def main(argv): process_num = 1 thread_num = 1 try: - opts, args = getopt.getopt(argv,"ht:p:",["threadN=","processN="]) + opts, args = getopt.getopt(argv, "ht:p:", ["threadN=", "processN="]) except getopt.GetoptError: - print('fusetest.py -t -p ') + print("fusetest.py -t -p ") sys.exit(2) for opt, arg in opts: - if opt == '-h': - print ('fusetest.py -t -p ') + if opt == "-h": + print("fusetest.py -t -p ") sys.exit() elif opt in ("-t", "--ifile"): thread_num = int(arg) elif opt in ("-p", "--ofile"): process_num = int(arg) - + cascade_process = multiprocessing.Process(target=initial_setup) cascade_process.start() cascade_process.join() - mnt_dir = pjoin(basename,"testdir") # use build/src/service/fuse/testdir as mounting point + mnt_dir = pjoin( + basename, "testdir" + ) # use build/src/service/fuse/testdir as mounting point fuse_process = multiprocessing.Process(target=cascade_fuse_mount, args=(mnt_dir,)) fuse_process.start() - fuse_process.join() # used when fuse not run in foreground in fuse_client.cpp - + fuse_process.join() # used when fuse not run in foreground in fuse_client.cpp + # multi-processing test cases test_processes = [] for i in range(process_num): run_id = "process_" + str(i) - proc = multiprocessing.Process(target=fuse_test_cases, args=(mnt_dir,run_id,)) + proc = multiprocessing.Process( + target=fuse_test_cases, + args=( + mnt_dir, + run_id, + ), + ) test_processes.append(proc) proc.start() for proc in test_processes: @@ -235,7 +347,13 @@ def main(argv): test_threads = [] for i in range(thread_num): run_id = "thread_" + str(i) - thread = threading.Thread(target=fuse_test_cases, args=(mnt_dir,run_id,)) + thread = threading.Thread( + target=fuse_test_cases, + args=( + mnt_dir, + run_id, + ), + ) test_threads.append(thread) thread.start() for thread in test_threads: @@ -244,5 +362,6 @@ def main(argv): # fuse_process.terminate() cascade_fuse_umount(mnt_dir) + if __name__ == "__main__": - main(sys.argv[1:]) + main(sys.argv[1:]) diff --git a/src/service/fuse/util.py b/src/service/fuse/util.py index 45b239a8..1548b870 100644 --- a/src/service/fuse/util.py +++ b/src/service/fuse/util.py @@ -1,48 +1,47 @@ #!/usr/bin/env python3 -import subprocess -import pytest +import itertools import os +import re import stat +import subprocess +import sys import time from os.path import join as pjoin -import sys -import re -import itertools + +import pytest basename = os.path.dirname(os.path.abspath(__file__)) fusermount3_dir = "/home/yy354/opt-dev/bin" def test_printcap(): - cmdline = base_cmdline + [ pjoin(basename, 'example', 'printcap') ] - proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE, - universal_newlines=True) + cmdline = base_cmdline + [pjoin(basename, "example", "printcap")] + proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE, universal_newlines=True) (stdout, _) = proc.communicate(30) assert proc.returncode == 0 proto = None caps = set() - for line in stdout.split('\n'): - if line.startswith('\t'): + for line in stdout.split("\n"): + if line.startswith("\t"): caps.add(line.strip()) continue - hit = re.match(r'Protocol version: (\d+)\.(\d+)$', line) + hit = re.match(r"Protocol version: (\d+)\.(\d+)$", line) if hit: proto = (int(hit.group(1)), int(hit.group(2))) return (proto, caps) -def wait_for_mount(mount_process, mnt_dir, - test_fn=os.path.ismount): +def wait_for_mount(mount_process, mnt_dir, test_fn=os.path.ismount): elapsed = 0 while elapsed < 20: if test_fn(mnt_dir): print(" -------- Fuse mount succeed -------") return True if mount_process.poll() is not None: - pytest.fail('file system process terminated prematurely') + pytest.fail("file system process terminated prematurely") time.sleep(0.1) elapsed += 0.1 pytest.fail("mountpoint failed to come up") @@ -56,28 +55,27 @@ def compare_dirs(output, expected): if not dir in output: return False return True - + def cleanup(mount_process, mnt_dir): # Don't bother trying Valgrind if things already went wrong - if 'bsd' in sys.platform or 'dragonfly' in sys.platform: - cmd = [ 'umount', '-f', mnt_dir ] + if "bsd" in sys.platform or "dragonfly" in sys.platform: + cmd = ["umount", "-f", mnt_dir] else: - cmd = [pjoin(basename, 'util', 'fusermount3'), - '-z', '-u', mnt_dir] - subprocess.call(cmd, stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT) + cmd = [pjoin(basename, "util", "fusermount3"), "-z", "-u", mnt_dir] + subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) mount_process.terminate() try: mount_process.wait(1) except subprocess.TimeoutExpired: mount_process.kill() + def umount(mnt_dir): - if 'bsd' in sys.platform or 'dragonfly' in sys.platform: - cmdline = [ 'umount', mnt_dir ] + if "bsd" in sys.platform or "dragonfly" in sys.platform: + cmdline = ["umount", mnt_dir] else: # fusermount3 will be setuid root, so we can only trace it with # valgrind if we're root @@ -85,21 +83,17 @@ def umount(mnt_dir): cmdline = base_cmdline else: cmdline = [] - cmdline = cmdline + [ pjoin(fusermount3_dir, 'fusermount3'), - '-z', '-u', mnt_dir ] + cmdline = cmdline + [pjoin(fusermount3_dir, "fusermount3"), "-z", "-u", mnt_dir] subprocess.check_call(cmdline) assert not os.path.ismount(mnt_dir) - - - def safe_sleep(secs): - '''Like time.sleep(), but sleep for at least *secs* + """Like time.sleep(), but sleep for at least *secs* `time.sleep` may sleep less than the given period if a signal is received. This function ensures that we sleep for at least the desired time. - ''' + """ now = time.time() end = now + secs @@ -109,26 +103,30 @@ def safe_sleep(secs): def powerset(iterable): - s = list(iterable) - return itertools.chain.from_iterable( - itertools.combinations(s, r) for r in range(len(s)+1)) + s = list(iterable) + return itertools.chain.from_iterable( + itertools.combinations(s, r) for r in range(len(s) + 1) + ) # Use valgrind if requested -if os.environ.get('TEST_WITH_VALGRIND', 'no').lower().strip() \ - not in ('no', 'false', '0'): - base_cmdline = [ 'valgrind', '-q', '--' ] +if os.environ.get("TEST_WITH_VALGRIND", "no").lower().strip() not in ( + "no", + "false", + "0", +): + base_cmdline = ["valgrind", "-q", "--"] else: base_cmdline = [] # Try to use local fusermount3 -os.environ['PATH'] = '%s:%s' % (pjoin(basename, 'util'), os.environ['PATH']) +os.environ["PATH"] = "%s:%s" % (pjoin(basename, "util"), os.environ["PATH"]) # Put example binaries on PATH -os.environ['PATH'] = '%s:%s' % (pjoin(basename, 'example'), os.environ['PATH']) +os.environ["PATH"] = "%s:%s" % (pjoin(basename, "example"), os.environ["PATH"]) try: (fuse_proto, fuse_caps) = test_printcap() except: # Rely on test to raise error - fuse_proto = (0,0) + fuse_proto = (0, 0) fuse_caps = set() diff --git a/src/service/server.cpp b/src/service/server.cpp index c7850d45..fb1c4960 100644 --- a/src/service/server.cpp +++ b/src/service/server.cpp @@ -1,10 +1,11 @@ #include "server.hpp" #include "cascade/cascade.hpp" +#include "cascade/object.hpp" #include "cascade/service.hpp" #include "cascade/service_types.hpp" -#include "cascade/object.hpp" +#include #include #include #include @@ -15,7 +16,36 @@ using namespace derecho::cascade; +void terminate() { + // wait for service to quit. + Service::shutdown(false); + dbg_default_trace("shutdown service gracefully"); + // you can do something here to parallel the destructing process. + Service::wait(); + dbg_default_trace("Finish shutdown."); +} + +void signal_handler(int signum) { + dbg_default_trace("received interrupt signal {}", signum); + + terminate(); + exit(signum); +} + int main(int argc, char** argv) { + // check for signal_arg + bool use_signal = false; + for(int i = 0; i < argc; ++i) { + printf("Argument %d : %s\n", i, argv[i]); + if(strcmp(argv[i], "--signal") == 0) { + use_signal = true; + } + } + // set proc name if(prctl(PR_SET_NAME, PROC_NAME, 0, 0, 0) != 0) { dbg_default_warn("Cannot set proc name to {}.", PROC_NAME); @@ -47,18 +77,18 @@ int main(int argc, char** argv) { meta_factory, vcss_factory, pcss_factory, tcss_factory); dbg_default_trace("started service, waiting till it ends."); - std::cout << "Press Enter to Shutdown." << std::endl; - std::cin.get(); - // wait for service to quit. - Service::shutdown(false); - dbg_default_trace("shutdown service gracefully"); - // you can do something here to parallel the destructing process. - Service::wait(); - dbg_default_trace("Finish shutdown."); + + if(use_signal) { + printf("Send SIGINT (Ctrl+C) to Shutdown.\n"); + signal(SIGINT, signal_handler); + while(true) { + sleep(60); + } + } else { + printf("Press Enter to Shutdown.\n"); + std::cin.get(); + terminate(); + } return 0; }