Merge remote-tracking branch 'baidu_tera/master' into rb.m.ng

lylei · lylei · commit 0323460fa55d · 2015-11-04T16:58:49.000+08:00
diff --git a/src/io/tablet_io.cc b/src/io/tablet_io.cc
@@ -98,7 +98,12 @@ std::string TabletIO::GetTableName() const {
 
 std::string TabletIO::GetTablePath() const {
     if (!m_tablet_path.empty()) {
-        return m_tablet_path.substr(FLAGS_tera_tabletnode_path_prefix.size());
+        std::string path =
+            m_tablet_path.substr(FLAGS_tera_tabletnode_path_prefix.size());
+        if (path.at(0) == '/') {
+            path = path.substr(1);
+        }
+        return path;
     } else {
         return m_tablet_path;
     }
@@ -1482,7 +1487,6 @@ void TabletIO::SetupOptionsForLG() {
                 << ", buffer_size:" << lg_info->memtable_ldb_write_buffer_size
                 << ", block_size:"  << lg_info->memtable_ldb_block_size;
         }
-        LOG(INFO) << ", sst_size: " << lg_schema.sst_size() << " Bytes.";
         lg_info->sst_size = lg_schema.sst_size();
         m_ldb_options.sst_size = lg_schema.sst_size();
         exist_lg_list->insert(lg_i);
diff --git a/src/leveldb/db/db_impl.cc b/src/leveldb/db/db_impl.cc
@@ -1697,10 +1697,10 @@ void DBImpl::GetApproximateSizes(uint64_t* size, std::vector<uint64_t>* lgsize)
   // add mem&imm size
   if (size) {
     if (mem_) {
-      size += mem_->ApproximateMemoryUsage();
+      *size += mem_->ApproximateMemoryUsage();
     }
     if (imm_) {
-      size += imm_->ApproximateMemoryUsage();
+      *size += imm_->ApproximateMemoryUsage();
     }
   }
 }
diff --git a/src/leveldb/include/leveldb/env.h b/src/leveldb/include/leveldb/env.h
@@ -379,7 +379,6 @@ class EnvWrapper : public Env {
     return target_->NewLogger(fname, result);
   }
   virtual void SetLogger(Logger* logger) {
-    Logger::SetDefaultLogger(logger);
     return target_->SetLogger(logger);
   }
   uint64_t NowMicros() {
diff --git a/src/leveldb/util/env_posix.cc b/src/leveldb/util/env_posix.cc
@@ -795,6 +795,7 @@ class PosixEnv : public Env {
     }
   }
   virtual void SetLogger(Logger* logger) {
+    Logger::SetDefaultLogger(logger);
     info_log_ = logger;
     thread_pool_.SetLogger(logger);
   }
diff --git a/src/master/master_impl.cc b/src/master/master_impl.cc
@@ -92,6 +92,7 @@ DECLARE_bool(tera_ins_enabled);
 DECLARE_int64(tera_sdk_perf_counter_log_interval);
 
 DECLARE_bool(tera_acl_enabled);
+DECLARE_bool(tera_only_root_create_table);
 DECLARE_string(tera_master_gc_strategy);
 
 namespace tera {
@@ -155,6 +156,7 @@ bool MasterImpl::Init() {
         m_zk_adapter.reset(new FakeMasterZkAdapter(this, m_local_addr));
     }
 
+    LOG(INFO) << "[acl] " << (FLAGS_tera_acl_enabled ? "enabled" : "disabled");
     SetMasterStatus(kIsSecondary);
     m_thread_pool->AddTask(boost::bind(&MasterImpl::InitAsync, this));
     return true;
@@ -462,16 +464,37 @@ bool MasterImpl::IsRootUser(const std::string& token) {
     return m_user_manager->UserNameToToken("root") == token;
 }
 
+// user is admin or user is in admin_group
+bool MasterImpl::CheckUserPermissionOnTable(const std::string& token, TablePtr table) {
+   std::string group_name = table->GetSchema().admin_group();
+   std::string user_name = m_user_manager->TokenToUserName(token);
+   return (m_user_manager->IsUserInGroup(user_name, group_name)
+           || (table->GetSchema().admin() == m_user_manager->TokenToUserName(token)));
+}
+
+template <typename Request>
+bool MasterImpl::HasPermissionOnTable(const Request* request, TablePtr table) {
+    if (!FLAGS_tera_acl_enabled
+        || IsRootUser(request->user_token())
+        || ((table->GetSchema().admin_group() == "") && (table->GetSchema().admin() == ""))
+        || (request->has_user_token()
+            && CheckUserPermissionOnTable(request->user_token(), table))) {
+        return true;
+
+    }
+    return false;
+}
+
 template <typename Request, typename Response, typename Callback>
-bool MasterImpl::HasTablePermission(const Request* request, Response* response,
-                                    Callback* done, TablePtr table, const char* operate) {
+bool MasterImpl::HasPermissionOrReturn(const Request* request, Response* response,
+                                       Callback* done, TablePtr table, const char* operate) {
     // check permission
-    if (!FLAGS_tera_acl_enabled
-        || IsRootUser(request->user_token())) {
-        LOG(INFO) << "[acl] is acl enabled: " << FLAGS_tera_acl_enabled;
+    if (HasPermissionOnTable(request, table)) {
         return true;
     } else {
-        LOG(INFO) << "[acl] fail to " << operate;
+        std::string token = request->has_user_token() ? request->user_token() : "";
+        LOG(INFO) << "[acl] " << m_user_manager->TokenToUserName(token)
+                  << ":" << token << " fail to " << operate;
         response->set_sequence_id(request->sequence_id());
         response->set_status(kNotPermission);
         done->Run();
@@ -661,11 +684,13 @@ void MasterImpl::CreateTable(const CreateTableRequest* request,
             done->Run();
             return;
         }
-        if (FLAGS_tera_acl_enabled && !IsRootUser(request->user_token())) {
-            response->set_sequence_id(request->sequence_id());
-            response->set_status(kNotPermission);
-            done->Run();
-            return;
+        if (FLAGS_tera_acl_enabled
+            && !IsRootUser(request->user_token())
+            && FLAGS_tera_only_root_create_table) {
+                response->set_sequence_id(request->sequence_id());
+                response->set_status(kNotPermission);
+                done->Run();
+                return;
         }
         if (!request->schema().alias().empty()) {
             bool alias_exist = false;
@@ -786,7 +811,7 @@ void MasterImpl::DeleteTable(const DeleteTableRequest* request,
         done->Run();
         return;
     }
-    if (!HasTablePermission(request, response, done, table, "delete table")) {
+    if (!HasPermissionOrReturn(request, response, done, table, "delete table")) {
         return;
     }
 
@@ -842,7 +867,7 @@ void MasterImpl::DisableTable(const DisableTableRequest* request,
         done->Run();
         return;
     }
-    if (!HasTablePermission(request, response, done, table, "disable table")) {
+    if (!HasPermissionOrReturn(request, response, done, table, "disable table")) {
         return;
     }
 
@@ -901,7 +926,7 @@ void MasterImpl::EnableTable(const EnableTableRequest* request,
         done->Run();
         return;
     }
-    if (!HasTablePermission(request, response, done, table, "enable table")) {
+    if (!HasPermissionOrReturn(request, response, done, table, "enable table")) {
         return;
     }
 
@@ -952,7 +977,7 @@ void MasterImpl::UpdateTable(const UpdateTableRequest* request,
         done->Run();
         return;
     }
-    if (!HasTablePermission(request, response, done, table, "update table")) {
+    if (!HasPermissionOrReturn(request, response, done, table, "update table")) {
         return;
     }
 
@@ -1074,11 +1099,19 @@ void MasterImpl::ShowTables(const ShowTablesRequest* request,
         TableMetaList* table_meta_list = response->mutable_table_meta_list();
         for (uint32_t i = 0; i < table_list.size(); ++i) {
             TablePtr table = table_list[i];
+            // if a user has NO permission on a table,
+            // he/she should not notice this table
+            if (!HasPermissionOnTable(request, table)) {
+                continue;
+            }
             table->ToMeta(table_meta_list->add_meta());
         }
         TabletMetaList* tablet_meta_list = response->mutable_tablet_meta_list();
         for (uint32_t i = 0; i < tablet_list.size(); ++i) {
             TabletPtr tablet = tablet_list[i];
+            if (!HasPermissionOnTable(request, tablet->GetTable())) {
+                continue;
+            }
             TabletMeta meta;
             tablet->ToMeta(&meta);
             tablet_meta_list->add_meta()->CopyFrom(meta);
@@ -1128,6 +1161,9 @@ void MasterImpl::ShowTabletNodes(const ShowTabletNodesRequest* request,
         std::vector<TabletPtr> tablet_list;
         m_tablet_manager->FindTablet(request->addr(), &tablet_list);
         for (size_t i = 0; i < tablet_list.size(); ++i) {
+            if (!HasPermissionOnTable(request, tablet_list[i]->GetTable())) {
+                continue;
+            }
             TabletMeta* meta = response->mutable_tabletmeta_list()->add_meta();
             TabletCounter* counter = response->mutable_tabletmeta_list()->add_counter();
             tablet_list[i]->ToMeta(meta);
@@ -3460,12 +3496,13 @@ void MasterImpl::QueryTabletNodeCallback(std::string addr, QueryRequest* request
                 ClearUnusedSnapshots(tablet, meta);
                 VLOG(30) << "[query] " << tablet;
             } else {
-                VLOG(30) << "fail to match tablet: " << meta.table_name()
+                LOG(WARNING) << "fail to match tablet: " << meta.table_name()
                     << ", path: " << meta.path()
                     << ", range: [" << DebugString(key_start)
                     << ", " << DebugString(key_end)
                     << "], size: " << meta.size()
-                    << ", addr: " << meta.server_addr();
+                    << ", addr: " << meta.server_addr()
+                    << ", tablet: " << tablet;
             }
         }
 
diff --git a/src/master/master_impl.h b/src/master/master_impl.h
@@ -528,9 +528,15 @@ class MasterImpl {
 
     bool IsRootUser(const std::string& token);
 
+    bool CheckUserPermissionOnTable(const std::string& token, TablePtr table);
+
+    template <typename Request>
+    bool HasPermissionOnTable(const Request* request, TablePtr table);
+
     template <typename Request, typename Response, typename Callback>
-    bool HasTablePermission(const Request* request, Response* response,
-                            Callback* done, TablePtr table, const char* operate);
+    bool HasPermissionOrReturn(const Request* request, Response* response,
+                               Callback* done, TablePtr table, const char* operate);
+
     void FillAlias(const std::string& key, const std::string& value);
 private:
     mutable Mutex m_status_mutex;
diff --git a/src/sdk/client_impl.cc b/src/sdk/client_impl.cc
@@ -511,43 +511,24 @@ bool ClientImpl::ShowTablesInfo(const string& name,
                                 TableMeta* meta,
                                 TabletMetaList* tablet_list,
                                 ErrorCode* err) {
-    if (meta == NULL || tablet_list == NULL) {
-        return false;
-    }
-    tablet_list->Clear();
-    std::string internal_table_name;
-    if (!GetInternalTableName(name, err, &internal_table_name)) {
-        LOG(ERROR) << "faild to scan meta schema";
-        return false;
+    TableMetaList table_list;
+    bool result = DoShowTablesInfo(&table_list, tablet_list, name, err);
+    if (result) {
+        meta->CopyFrom(table_list.meta(0));
     }
-    master::MasterClient master_client(_cluster->MasterAddr());
-
-    ShowTablesRequest request;
-    ShowTablesResponse response;
-    request.set_sequence_id(0);
-    request.set_start_table_name(internal_table_name);
-    request.set_max_table_num(1);
-    request.set_user_token(GetUserToken(_user_identity, _user_passcode));
-
-    if (master_client.ShowTables(&request, &response) &&
-        response.status() == kMasterOk) {
-        if (response.table_meta_list().meta_size() == 0) {
-            return false;
-        } else if (response.table_meta_list().meta(0).table_name() != internal_table_name) {
-            return false;
-        }
-        meta->CopyFrom(response.table_meta_list().meta(0));
-        tablet_list->CopyFrom(response.tablet_meta_list());
-        return true;
-    }
-    LOG(ERROR) << "fail to show table info: " << name;
-    err->SetFailed(ErrorCode::kSystem, StatusCodeToString(response.status()));
-    return false;
+    return result;
 }
 
 bool ClientImpl::ShowTablesInfo(TableMetaList* table_list,
                                 TabletMetaList* tablet_list,
                                 ErrorCode* err) {
+    return DoShowTablesInfo(table_list, tablet_list, "", err);
+}
+
+bool ClientImpl::DoShowTablesInfo(TableMetaList* table_list,
+                                  TabletMetaList* tablet_list,
+                                  const string& table_name,
+                                  ErrorCode* err) {
     if (table_list == NULL || tablet_list == NULL) {
         return false;
     }
@@ -556,14 +537,17 @@ bool ClientImpl::ShowTablesInfo(TableMetaList* table_list,
 
     master::MasterClient master_client(_cluster->MasterAddr());
     std::string start_tablet_key;
-    std::string start_table_name;
+    std::string start_table_name = table_name;
     bool has_more = true;
     bool has_error = false;
     bool table_meta_copied = false;
     std::string err_msg;
     while(has_more && !has_error) {
         ShowTablesRequest request;
         ShowTablesResponse response;
+        if (!table_name.empty()) {
+            request.set_max_table_num(1);
+        }
         request.set_start_table_name(start_table_name);
         request.set_start_tablet_key(start_tablet_key);
         request.set_max_tablet_num(FLAGS_tera_sdk_show_max_num); //tablets be fetched at most in one RPC
@@ -614,17 +598,18 @@ bool ClientImpl::ShowTablesInfo(TableMetaList* table_list,
 
     if (has_error) {
         LOG(ERROR) << "fail to show table info.";
-        err->SetFailed(ErrorCode::kSystem, err_msg);
+        if (err != NULL) {
+            err->SetFailed(ErrorCode::kSystem, err_msg);
+        }
         return false;
     }
     return true;
 }
 
-
 bool ClientImpl::ShowTabletNodesInfo(const string& addr,
-                                    TabletNodeInfo* info,
-                                    TabletMetaList* tablet_list,
-                                    ErrorCode* err) {
+                                     TabletNodeInfo* info,
+                                     TabletMetaList* tablet_list,
+                                     ErrorCode* err) {
     if (info == NULL || tablet_list == NULL) {
         return false;
     }
@@ -655,7 +640,7 @@ bool ClientImpl::ShowTabletNodesInfo(const string& addr,
 }
 
 bool ClientImpl::ShowTabletNodesInfo(std::vector<TabletNodeInfo>* infos,
-                                    ErrorCode* err) {
+                                     ErrorCode* err) {
     if (infos == NULL) {
         return false;
     }
diff --git a/src/sdk/client_impl.h b/src/sdk/client_impl.h
@@ -140,6 +140,13 @@ class ClientImpl : public Client {
     bool CheckReturnValue(StatusCode status, std::string& reason, ErrorCode* err);
     bool GetInternalTableName(const std::string& table_name, ErrorCode* err,
                               std::string* internal_table_name);
+
+    /// show all tables info: `table_name' should be an empty string
+    /// show a single table info: `table_name' should be the table name
+    bool DoShowTablesInfo(TableMetaList* table_list,
+                          TabletMetaList* tablet_list,
+                          const string& table_name,
+                          ErrorCode* err);
 private:
     ClientImpl(const ClientImpl&);
     void operator=(const ClientImpl&);
diff --git a/src/tabletnode/tabletnode_impl.h b/src/tabletnode/tabletnode_impl.h
@@ -14,7 +14,6 @@
 #include "proto/master_rpc.pb.h"
 #include "proto/tabletnode.pb.h"
 #include "proto/tabletnode_rpc.pb.h"
-#include "proto/table_meta.pb.h"
 #include "tabletnode/rpc_compactor.h"
 #include "tabletnode/tabletnode_sysinfo.h"
 #include "utils/rpc_timer_list.h"
diff --git a/src/teracli_main.cc b/src/teracli_main.cc

Original file line number	Diff line number	Diff line change
`@@ -1697,10 +1697,10 @@ void DBImpl::GetApproximateSizes(uint64_t* size, std::vector<uint64_t>* lgsize)`
`1697`	`1697`	`// add mem&imm size`
`1698`	`1698`	`if (size) {`
`1699`	`1699`	`if (mem_) {`
`1700`		`- size += mem_->ApproximateMemoryUsage();`
	`1700`	`+ *size += mem_->ApproximateMemoryUsage();`
`1701`	`1701`	`}`
`1702`	`1702`	`if (imm_) {`
`1703`		`- size += imm_->ApproximateMemoryUsage();`
	`1703`	`+ *size += imm_->ApproximateMemoryUsage();`
`1704`	`1704`	`}`
`1705`	`1705`	`}`
`1706`	`1706`	`}`
Original file line number	Diff line number	Diff line change
`@@ -379,7 +379,6 @@ class EnvWrapper : public Env {`
`379`	`379`	`return target_->NewLogger(fname, result);`
`380`	`380`	`}`
`381`	`381`	`virtual void SetLogger(Logger* logger) {`
`382`		`- Logger::SetDefaultLogger(logger);`
`383`	`382`	`return target_->SetLogger(logger);`
`384`	`383`	`}`
`385`	`384`	`uint64_t NowMicros() {`
Original file line number	Diff line number	Diff line change
`@@ -795,6 +795,7 @@ class PosixEnv : public Env {`
`795`	`795`	`}`
`796`	`796`	`}`
`797`	`797`	`virtual void SetLogger(Logger* logger) {`
	`798`	`+ Logger::SetDefaultLogger(logger);`
`798`	`799`	`info_log_ = logger;`
`799`	`800`	`thread_pool_.SetLogger(logger);`
`800`	`801`	`}`