fix: prevent deadlocks when instrumenting with clippy-tracing

Two deadlock sources exist when clippy-tracing instruments the codebase:
1. Logger::update() called open_file_nonblock() while holding the LOGGER
   mutex. When instrumented, that function calls log::trace!() which
   re-enters Logger::log() and attempts to acquire the same non-reentrant
   mutex, causing a deadlock on startup.
2. Signal handlers (extern C fns) were instrumented by clippy-tracing,
   adding mutex acquisitions in async-signal-unsafe contexts. When a
   signal interrupted a thread already holding the LOGGER mutex, the
   handler would deadlock trying to acquire it again.
Fix (1) by moving the open_file_nonblock() call before the mutex is
acquired in Logger::update(). Fix (2) by teaching clippy-tracing to
skip functions with an explicit ABI (e.g. extern C), since these are
typically signal handlers or FFI callbacks that must not acquire locks.
Closes #5807

Author: snigenigmatic

src/clippy-tracing/src/main.rs

@@ -128,10 +128,7 @@ macro_rules! create_check_visitor_function {
     ($func_name:ident, $item:ident) => {
         fn $func_name(&mut self, i: &syn::$item) {
             let attr = check_attributes(&i.attrs);
-            if !attr.instrumented
-                && !attr.test
-                && i.sig.constness.is_none()
-                && i.sig.abi.is_none()
+            if !attr.instrumented && !attr.test && i.sig.constness.is_none() && i.sig.abi.is_none()
             {
                 self.0 = Some(i.span());
             } else {
@@ -166,10 +163,7 @@ macro_rules! create_fix_visitor_function {
         fn $func_name(&mut self, i: &syn::$item) {
             let attr = check_attributes(&i.attrs);
 
-            if !attr.instrumented
-                && !attr.test
-                && i.sig.constness.is_none()
-                && i.sig.abi.is_none()
+            if !attr.instrumented && !attr.test && i.sig.constness.is_none() && i.sig.abi.is_none()
             {
                 let line = i.span().start().line;
 

src/clippy-tracing/tests/integration_tests.rs

@@ -431,7 +431,8 @@ fn check_skips_extern_c_fn() {
 
 #[test]
 fn fix_skips_extern_c_fn_in_impl() {
-    const GIVEN: &str = "struct S;\nimpl S {\n    extern \"C\" fn callback() {}\n    fn normal() {}\n}";
+    const GIVEN: &str =
+        "struct S;\nimpl S {\n    extern \"C\" fn callback() {}\n    fn normal() {}\n}";
     const EXPECTED: &str = "struct S;\nimpl S {\n    extern \"C\" fn callback() {}\n    #[log_instrument::instrument]\n    fn normal() {}\n}";
     fix(GIVEN, EXPECTED, None);
 }