diff --git a/CMakeLists.txt b/CMakeLists.txt index 6052748601..1ee4fbccd0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,12 @@ set( INSTALLER_APP_ID "68ad7005-8eee-49c9-95ce-9eed97e5b347" ) set( CMAKE_CXX_STANDARD 14 ) set( CMAKE_CXX_STANDARD_REQUIRED ON ) set( CMAKE_CXX_EXTENSIONS OFF ) - +option(LOAD_QUERY_TXID_PLUGIN OFF) +if(${LOAD_QUERY_TXID_PLUGIN} STREQUAL "ON") + add_definitions(-DQUERY_TXID_PLUGIN_ABLE) + set( QUERY_TXID graphene_query_txid) +endif() +MESSAGE("the query_txid plugin is ${LOAD_QUERY_TXID_PLUGIN}") # http://stackoverflow.com/a/18369825 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) diff --git a/libraries/app/CMakeLists.txt b/libraries/app/CMakeLists.txt index bf4f5c2b00..29388cd824 100644 --- a/libraries/app/CMakeLists.txt +++ b/libraries/app/CMakeLists.txt @@ -13,7 +13,7 @@ add_library( graphene_app ) # need to link graphene_debug_witness because plugins aren't sufficiently isolated #246 -target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_grouped_orders graphene_chain fc graphene_db graphene_net graphene_utilities graphene_debug_witness ) +target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_grouped_orders graphene_chain fc graphene_db graphene_net graphene_utilities graphene_debug_witness ${QUERY_TXID}) target_include_directories( graphene_app PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../egenesis/include" ) diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp index 6c12eeecf5..a34bb8ceac 100644 --- a/libraries/app/database_api.cpp +++ b/libraries/app/database_api.cpp @@ -42,7 +42,9 @@ #include #include - +#ifdef QUERY_TXID_PLUGIN_ABLE +#include +#endif #define GET_REQUIRED_FEES_MAX_RECURSION 4 typedef std::map< std::pair, std::vector > market_queue_type; @@ -73,7 +75,7 @@ class database_api_impl : public std::enable_shared_from_this map> get_block_header_batch(const vector block_nums)const; optional get_block(uint32_t block_num)const; processed_transaction get_transaction( uint32_t block_num, uint32_t trx_in_block )const; - + optional get_transaction_by_txid(transaction_id_type txid)const; // Globals chain_property_object get_chain_properties()const; global_property_object get_global_properties()const; @@ -634,6 +636,10 @@ processed_transaction database_api::get_transaction( uint32_t block_num, uint32_ { return my->get_transaction( block_num, trx_in_block ); } +optional database_api::get_transaction_by_txid(transaction_id_type txid)const +{ + return my->get_transaction_by_txid(txid); +} optional database_api::get_recent_transaction_by_id( const transaction_id_type& id )const { @@ -652,6 +658,40 @@ processed_transaction database_api_impl::get_transaction(uint32_t block_num, uin return opt_block->transactions[trx_num]; } +optional database_api_impl::get_transaction_by_txid(transaction_id_type txid)const +{ +#ifdef QUERY_TXID_PLUGIN_ABLE + auto &txid_index = _db.get_index_type().indices().get(); + auto itor = txid_index.find(txid); + if (itor == txid_index.end()) { + std::string txid_str(txid); + auto result = query_txid::query_txid_plugin::query_trx_by_id(txid_str); + if (result) { + const auto &trx_entry = *result; + auto opt_block = _db.fetch_block_by_number(trx_entry.block_num); + FC_ASSERT(opt_block); + FC_ASSERT(opt_block->transactions.size() > trx_entry.trx_in_block); + optional res = opt_block->transactions[trx_entry.trx_in_block]; + return res; + } + return {}; + } else { + const auto &dpo = _db.get_dynamic_global_properties(); + if (itor->block_num <= dpo.last_irreversible_block_num) { + const auto &trx_entry = *itor; + auto opt_block = _db.fetch_block_by_number(trx_entry.block_num); + FC_ASSERT(opt_block); + FC_ASSERT(opt_block->transactions.size() > trx_entry.trx_in_block); + optional res = opt_block->transactions[trx_entry.trx_in_block]; + return res; + } else { + return {}; + } + } +#endif + return {}; +} + ////////////////////////////////////////////////////////////////////// // // // Globals // diff --git a/libraries/app/include/graphene/app/database_api.hpp b/libraries/app/include/graphene/app/database_api.hpp index 13734e751b..64b11d32d5 100644 --- a/libraries/app/include/graphene/app/database_api.hpp +++ b/libraries/app/include/graphene/app/database_api.hpp @@ -41,7 +41,7 @@ #include #include #include - +#include #include #include @@ -232,7 +232,7 @@ class database_api * @brief used to fetch an individual transaction. */ processed_transaction get_transaction( uint32_t block_num, uint32_t trx_in_block )const; - + optional get_transaction_by_txid(transaction_id_type txid)const; /** * If the transaction has not expired, this method will return the transaction for the given ID or * it will return NULL if it is not known. Just because it is not known does not mean it wasn't @@ -853,7 +853,8 @@ FC_API(graphene::app::database_api, (get_block) (get_transaction) (get_recent_transaction_by_id) - + (get_transaction_by_txid) + // Globals (get_chain_properties) (get_global_properties) diff --git a/libraries/chain/include/graphene/chain/transcation_entry_object.hpp b/libraries/chain/include/graphene/chain/transcation_entry_object.hpp new file mode 100644 index 0000000000..13719e808b --- /dev/null +++ b/libraries/chain/include/graphene/chain/transcation_entry_object.hpp @@ -0,0 +1,42 @@ +#pragma once +#include +//#include +//#include +#include +#include + +namespace graphene { namespace chain { + + class trx_entry_object : public abstract_object + { + public: + static const uint8_t space_id = implementation_ids; + static const uint8_t type_id = impl_trx_entry_history_object_type; + + trx_entry_object(){} + + transaction_id_type txid; + uint32_t block_num; + uint32_t trx_in_block; + }; + + // struct by_id; + struct by_txid; + struct by_blocknum; + + typedef multi_index_container< + trx_entry_object, + indexed_by< + ordered_unique< tag, member< object, object_id_type, &object::id > >, + ordered_unique< tag, member< trx_entry_object, transaction_id_type, &trx_entry_object::txid > >, + ordered_non_unique< tag, member< trx_entry_object, uint32_t, &trx_entry_object::block_num > > + > + + > trx_entry_multi_index_type; + + typedef generic_index trx_entry_index; + +} } // graphene::chain + +FC_REFLECT_DERIVED( graphene::chain::trx_entry_object, (graphene::chain::object), + (txid)(block_num)(trx_in_block)) \ No newline at end of file diff --git a/libraries/chain/include/graphene/chain/types.hpp b/libraries/chain/include/graphene/chain/types.hpp index ec3e788219..872ee6e623 100644 --- a/libraries/chain/include/graphene/chain/types.hpp +++ b/libraries/chain/include/graphene/chain/types.hpp @@ -45,4 +45,5 @@ GRAPHENE_DEFINE_IDS(chain, implementation_ids, impl_, (special_authority) (buyback) (fba_accumulator) - (collateral_bid)) + (collateral_bid) + (trx_entry_history)) diff --git a/libraries/plugins/CMakeLists.txt b/libraries/plugins/CMakeLists.txt index caacb8bd53..54bd932c14 100644 --- a/libraries/plugins/CMakeLists.txt +++ b/libraries/plugins/CMakeLists.txt @@ -7,3 +7,4 @@ add_subdirectory( delayed_node ) add_subdirectory( debug_witness ) add_subdirectory( snapshot ) add_subdirectory( es_objects ) +add_subdirectory( query_txid ) \ No newline at end of file diff --git a/libraries/plugins/query_txid/CMakeLists.txt b/libraries/plugins/query_txid/CMakeLists.txt new file mode 100644 index 0000000000..73f09b3ea1 --- /dev/null +++ b/libraries/plugins/query_txid/CMakeLists.txt @@ -0,0 +1,24 @@ +file(GLOB HEADERS "include/graphene/query_txid/*.hpp") + +add_library( graphene_query_txid + query_txid_plugin.cpp + ) +find_path(LevelDB_INCLUDE_PATH NAMES leveldb/db.h leveldb/write_batch.h) +find_library(LevelDB_LIBRARY NAMES libleveldb.a) +find_library(Snappy_LIBRARY NAMES libsnappy.a) + +if(LevelDB_INCLUDE_PATH AND LevelDB_LIBRARY AND Snappy_LIBRARY) + target_link_libraries( graphene_query_txid graphene_chain graphene_app ${LevelDB_LIBRARY} ${Snappy_LIBRARY}) + target_include_directories( graphene_query_txid + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ${LevelDB_INCLUDE_PATH}) + install( TARGETS + graphene_query_txid + + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + ) +else(LevelDB_INCLUDE_PATH AND LevelDB_LIBRARY AND Snappy_LIBRARY) + message(FATAL_ERROR "You need leveldb and snappy") +endif() + diff --git a/libraries/plugins/query_txid/include/graphene/query_txid/query_txid_plugin.hpp b/libraries/plugins/query_txid/include/graphene/query_txid/query_txid_plugin.hpp new file mode 100644 index 0000000000..9adad6acf6 --- /dev/null +++ b/libraries/plugins/query_txid/include/graphene/query_txid/query_txid_plugin.hpp @@ -0,0 +1,39 @@ +#pragma once +#include +#include +#include + + +namespace graphene +{ +namespace query_txid +{ + +using namespace chain; +namespace detail +{ +class query_txid_plugin_impl; +} +class query_txid_plugin : public graphene::app::plugin +{ + public: + query_txid_plugin(); + virtual ~query_txid_plugin(); + + std::string plugin_name() const override; + + virtual void plugin_set_program_options( + boost::program_options::options_description &cli, + boost::program_options::options_description &cfg) override; + + virtual void plugin_initialize(const boost::program_options::variables_map &options) override; + virtual void plugin_startup() override; + + static optional query_trx_by_id(std::string txid); + + friend class detail::query_txid_plugin_impl; + + std::unique_ptr my; +}; +} // namespace query_txid +} // namespace graphene diff --git a/libraries/plugins/query_txid/query_txid_plugin.cpp b/libraries/plugins/query_txid/query_txid_plugin.cpp new file mode 100644 index 0000000000..aba592088c --- /dev/null +++ b/libraries/plugins/query_txid/query_txid_plugin.cpp @@ -0,0 +1,210 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace graphene{ namespace query_txid{ + +namespace detail +{ +using namespace leveldb; +class query_txid_plugin_impl +{ + public: + query_txid_plugin_impl(query_txid_plugin &_plugin) + : _self(_plugin) + { + } + ~query_txid_plugin_impl() + { + } + + void collect_txid_index(const signed_block &b); + + graphene::chain::database &database() + { + return _self.database(); + } + void init(); + static optional query_trx_by_id(std::string txid); + std::string db_path = "trx_entry.db"; + uint64_t limit_batch = 1000; //limit of leveldb batch + + private: + query_txid_plugin &_self; + + fc::signal sig_db_write; + fc::signal sig_remove; + + static leveldb::DB *leveldb; + void consume_block(); //Consume block + void remove_trx_index(const uint64_t trx_entry_id); //Remove trx_index in db +}; +leveldb::DB *query_txid_plugin_impl::leveldb = nullptr; + +void query_txid_plugin_impl::init() +{ + try { + //Create leveldb + leveldb::Options options; + options.create_if_missing = true; + leveldb::Status s = leveldb::DB::Open(options, db_path, &leveldb); + + // Respond to the sig_db_write signale + sig_db_write.connect([&]() { consume_block(); }); + sig_remove.connect([&](const uint64_t trx_entry_id) { remove_trx_index(trx_entry_id); }); + } + FC_LOG_AND_RETHROW() +} +optional query_txid_plugin_impl::query_trx_by_id(std::string txid) +{ + try { + if (leveldb == nullptr) return optional(); + std::string value; + leveldb::Status s = leveldb->Get(leveldb::ReadOptions(), txid, &value); + if (!s.ok()) return optional(); + std::vector data(value.begin(), value.end()); + auto result = fc::raw::unpack(data); + return result; + } + FC_LOG_AND_RETHROW() +} +void query_txid_plugin_impl::collect_txid_index(const signed_block &b) +{ + try { + graphene::chain::database &db = database(); + for (auto idx = 0; idx < b.transactions.size(); idx++) { + db.create([&](trx_entry_object &obj) { + obj.txid = b.transactions[idx].id(); + obj.block_num = b.block_num(); + obj.trx_in_block = idx; + }); + } + sig_db_write(); + } + FC_LOG_AND_RETHROW() +} +void query_txid_plugin_impl::consume_block() +{ + try { + graphene::chain::database &db = database(); + const auto &dpo = db.get_dynamic_global_properties(); + uint64_t irr_num = dpo.last_irreversible_block_num; + + const auto &trx_idx = db.get_index_type().indices(); + const auto &trx_bn_idx = trx_idx.get(); + if (trx_idx.begin() == trx_idx.end()) return; + auto itor_begin = trx_bn_idx.begin(); + auto itor_end = trx_bn_idx.lower_bound(irr_num); + auto number = std::distance(itor_begin,itor_end); + auto backupnum = number; + auto put_index = itor_begin->id.instance(); + while (number > limit_batch) { + leveldb::WriteBatch batch; + auto itor_backup = itor_begin; + for (auto idx = 0; idx < limit_batch; idx++) { + auto serialize = fc::raw::pack(*itor_begin); + std::string txid(itor_begin->txid); + batch.Put(txid, {serialize.data(), serialize.size()}); + put_index = itor_begin->id.instance(); + itor_begin++; + if (itor_begin == itor_end) break; + } + leveldb::WriteOptions write_options; + write_options.sync = true; + Status s = leveldb->Write(write_options, &batch); + if (!s.ok()) { + itor_begin = itor_backup; + put_index = itor_begin->id.instance(); + break; + } + number -= limit_batch; + } + if (backupnum > limit_batch) + sig_remove(put_index); + } + FC_LOG_AND_RETHROW() +} +void query_txid_plugin_impl::remove_trx_index(const uint64_t trx_entry_id) +{ + try { + graphene::chain::database &db = database(); + const auto &trx_idx = db.get_index_type().indices(); + //ilog("remove,${trx_ent_id},bengin: ${begin},end: ${end}",("trx_ent_id",trx_entry_id)("begin",trx_idx.begin()->id.instance())("end",trx_idx.rbegin()->id.instance())); + for (auto itor = trx_idx.begin(); itor != trx_idx.end();) { + auto backup_itr = itor; + ++itor; + if (itor->id.instance() < trx_entry_id) { + db.remove(*backup_itr); + } else { + break; + } + } + } + FC_LOG_AND_RETHROW() +} +} // namespace detail + +// -----------------------------------query_txid_plugin -------------------------------------- + +query_txid_plugin::query_txid_plugin() + : my(new detail::query_txid_plugin_impl(*this)) +{ +} + +query_txid_plugin::~query_txid_plugin() +{ +} + +std::string query_txid_plugin::plugin_name() const +{ + return "query_txid"; +} + +void query_txid_plugin::plugin_set_program_options( + boost::program_options::options_description &cli, + boost::program_options::options_description &cfg) +{ + cli.add_options()("query-txid-path", boost::program_options::value(), "Save the leveldb path of the transaction history")("limit-batch", boost::program_options::value(), "Number of records written to leveldb in batches"); + cfg.add(cli); +} + +void query_txid_plugin::plugin_initialize(const boost::program_options::variables_map &options) +{ + try { + ilog("query_txid plugin initialized"); + // Add the index of the trx_entry_index object table to the database + database().add_index>(); + // Respond to the apply_block signal + database().applied_block.connect([&](const signed_block &b) { my->collect_txid_index(b); }); + if (options.count("query-txid-path")) { + my->db_path = options["query-txid-path"].as(); + if (!fc::exists(my->db_path)) + fc::create_directories(my->db_path); + } + if (options.count("limit-batch")) { + my->limit_batch = options["limit-batch"].as(); + } + // Initialize the plugin instance + my->init(); + } + FC_LOG_AND_RETHROW() +} + +void query_txid_plugin::plugin_startup() +{ +} + +optional query_txid_plugin::query_trx_by_id(std::string txid) +{ + return detail::query_txid_plugin_impl::query_trx_by_id(txid); +} + +} // namespace query_txid +} // namespace graphene \ No newline at end of file diff --git a/programs/witness_node/CMakeLists.txt b/programs/witness_node/CMakeLists.txt index 4815879a40..3d2d62319c 100644 --- a/programs/witness_node/CMakeLists.txt +++ b/programs/witness_node/CMakeLists.txt @@ -12,7 +12,7 @@ endif() # We have to link against graphene_debug_witness because deficiency in our API infrastructure doesn't allow plugins to be fully abstracted #246 target_link_libraries( witness_node -PRIVATE graphene_app graphene_delayed_node graphene_account_history graphene_elasticsearch graphene_market_history graphene_grouped_orders graphene_witness graphene_chain graphene_debug_witness graphene_egenesis_full graphene_snapshot graphene_es_objects fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) +PRIVATE graphene_app graphene_delayed_node graphene_account_history graphene_elasticsearch graphene_market_history graphene_grouped_orders graphene_witness graphene_chain graphene_debug_witness graphene_egenesis_full graphene_snapshot graphene_es_objects ${QUERY_TXID} fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) install( TARGETS witness_node diff --git a/programs/witness_node/main.cpp b/programs/witness_node/main.cpp index c4748697c6..66a0456a1e 100644 --- a/programs/witness_node/main.cpp +++ b/programs/witness_node/main.cpp @@ -33,7 +33,9 @@ #include #include #include - +#ifdef QUERY_TXID_PLUGIN_ABLE +#include +#endif #include #include @@ -90,6 +92,9 @@ int main(int argc, char** argv) { auto snapshot_plug = node->register_plugin(); auto es_objects_plug = node->register_plugin(); auto grouped_orders_plug = node->register_plugin(); + #ifdef QUERY_TXID_PLUGIN_ABLE + auto querytxid_plug = node->register_plugin(); + #endif // add plugin options to config try