reimplement qspinlock using thread_local variables, and remove the dependence on photon vcpu (alibaba#965)

* reimplement qspinlock, to remove the dependece on photon vcpu,
so that it can be used in non-photon environment.

* Add lock perf for spinlock and mutex

---------

Co-authored-by: Coldwings <coldwings@me.com>

Author: lihuiba

examples/CMakeLists.txt

@@ -13,6 +13,9 @@ if (PHOTON_ENABLE_URING)
     target_compile_definitions(net-perf PRIVATE PHOTON_URING=1)
 endif()
 
+add_executable(lock-perf perf/lock-perf.cpp)
+target_link_libraries(lock-perf PRIVATE photon_static)
+
 add_executable(workpool-perf perf/workpool-perf.cpp)
 target_link_libraries(workpool-perf PRIVATE photon_static)
 

examples/perf/lock-perf.cpp

@@ -0,0 +1,59 @@
+/*
+Copyright 2022 The Photon Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <gflags/gflags.h>
+#include <photon/common/alog.h>
+#include <photon/photon.h>
+#include <photon/thread/thread11.h>
+
+#include <chrono>
+#include <thread>
+#include <vector>
+#include "photon/thread/thread.h"
+
+DEFINE_uint64(threads, 8, "threads requires lock");
+DEFINE_uint64(turn, 100000, "turns requiring lock for each thread");
+
+template <typename locktype>
+void lockperf(const char* name) {
+    std::vector<std::thread> ths;
+    locktype mtx;
+    for (size_t i = 0; i < FLAGS_threads; i++) {
+        ths.emplace_back([&] {
+            photon::init();
+            DEFER(photon::fini());
+            auto start = std::chrono::system_clock::now();
+            for (size_t j = 0; j < FLAGS_turn; j++) {
+                mtx.lock();
+                mtx.unlock();
+            }
+            auto end = std::chrono::system_clock::now();
+            LOG_TEMP("` time: `", name, (end - start).count());
+        });
+    }
+    for (auto& x : ths) x.join();
+}
+
+int main(int argc, char** arg) {
+    gflags::ParseCommandLineFlags(&argc, &arg, true);
+    log_output_level = ALOG_WARN;
+    photon::init();
+    DEFER(photon::fini());
+    lockperf<photon::spinlock>("spin");
+    lockperf<photon::qspinlock>("qspin");
+    lockperf<photon::mutex>("ticket");
+    return 0;
+}

thread/thread.cpp

@@ -569,8 +569,6 @@ namespace photon
             mee = &_default_event_engine;
         }
 
-        std::atomic<vcpu_base*> _qspin_next { nullptr };
-        std::atomic<bool> _qspin_got_lock { false };
         static spinlock vcpu_list_lock;    // lock when add, remove, iterate next
         static rwlock vcpu_list_rwlock;    // rlock when iterate, wlock when remove
         static intrusive_list<vcpu_t, false> pvcpu;
@@ -1629,33 +1627,44 @@ R"(
         serv.store(successor, std::memory_order_release);
     }
 
+    struct qspinlock::holder {
+        std::atomic<holder*> next { nullptr };
+        std::atomic<bool> got_lock { false };
+    };
+    static thread_local qspinlock::holder qslholder;
+    int qspinlock::try_lock() {
+        holder* expected = nullptr;
+        bool ok = _owner_tail.compare_exchange_strong(expected,
+                        &qslholder, std::memory_order_acq_rel);
+        return int(ok) - 1;
+    }
     int qspinlock::lock() {
-        auto vcpu = (vcpu_t*)get_vcpu();
-        assert(vcpu->_qspin_next == nullptr);
-        auto old_tail = (vcpu_t*)_owner_tail.exchange(vcpu, std::memory_order_acq_rel);
+        // forbid gcc to silly update h
+        auto h = &qslholder; asm volatile("": "+r"(h));
+        assert(h->next == nullptr);
+        auto old_tail = _owner_tail.exchange(h, std::memory_order_acq_rel);
         if (!old_tail) return 0;
 
-        vcpu->_qspin_got_lock.store(false, std::memory_order_relaxed);
-        assert(old_tail->_qspin_next.load(std::memory_order_acquire) == nullptr);
-        old_tail->_qspin_next.store(vcpu, std::memory_order_release);
+        h->got_lock.store(false, std::memory_order_relaxed);
+        assert(old_tail->next.load(std::memory_order_acquire) == nullptr);
+        old_tail->next.store(h, std::memory_order_release);
         do { spin_wait(); }
-        while (vcpu->_qspin_got_lock.load(std::memory_order_acquire) == false);
+        while (h->got_lock.load(std::memory_order_acquire) == false);
         return 0;
     }
-
     void qspinlock::unlock() {
-        auto vcpu = (vcpu_t*)get_vcpu();
+        // forbid gcc to silly update h
+        auto h = &qslholder; asm volatile("": "+r"(h));
         while(true) {
-            auto next = (vcpu_t*) vcpu->_qspin_next.load(std::memory_order_acquire);
+            auto next = h->next.load(std::memory_order_acquire);
             if (next) { // resume the next waiter, if there is one
-                vcpu->_qspin_next.store(nullptr, std::memory_order_release);
-                next->_qspin_got_lock.store(true, std::memory_order_release);
+                h->next.store(nullptr, std::memory_order_release);
+                next->got_lock.store(true, std::memory_order_release);
                 return;
             } else {    // do unlock if there is no waiter
-                vcpu_base* expected = vcpu;
-                bool ok = _owner_tail.compare_exchange_strong(expected,
-                                    nullptr, std::memory_order_acq_rel);
-                if (ok) return;
+                auto expected = h;
+                if (_owner_tail.compare_exchange_strong(expected,
+                    nullptr, std::memory_order_acq_rel)) return;
                 spin_wait();    // unlock failed, wait and try again
             }
         }

thread/thread.h

@@ -237,19 +237,15 @@ namespace photon
 
     class qspinlock {
     public:
-        int try_lock() {
-            vcpu_base* expected = nullptr;
-            bool ok = _owner_tail.compare_exchange_strong(expected,
-                            get_vcpu(), std::memory_order_acq_rel);
-            return int(ok) - 1;
-        }
+        int try_lock();
         int lock();
         bool locked() const {
             return _owner_tail.load(std::memory_order_acquire);
         }
         void unlock();
     protected:
-        std::atomic<vcpu_base*> _owner_tail { nullptr };
+        struct holder;
+        std::atomic<holder*> _owner_tail { nullptr };
     };
 
     class ticket_spinlock {