Skip to content

Commit c836aff

Browse files
tuaristuaris
committed
feat: improve FreeBSD platform support
Add comprehensive FreeBSD support to brpc, building on the existing partial FreeBSD code already present in the codebase. CMakeLists.txt: - Add NO_PTHREAD_MUTEX_HOOK on FreeBSD to prevent static init deadlock caused by bthread's pthread_mutex_lock interposition conflicting with libc++ ios_base::Init during early process initialization - Add -lexecinfo for backtrace support - Add FreeBSD-specific source files (platform_thread_freebsd.cc, sys_string_conversions_posix.cc) Source changes (all guarded by OS_FREEBSD, no impact on Linux/macOS): - butil/compat.h: Add FreeBSD kqueue path and pthread_getthreadid_np() - butil/process_util.cc: FreeBSD ReadCommandLine via ps (like macOS) - butil/fd_utility.cpp: kqueue fd handling - butil/logging.cc: Rename FLAGS_v/vmodule/minloglevel to avoid gflags duplicate registration with system glog on FreeBSD - brpc/socket.cpp: FreeBSD TCP_INFO support - bthread/: FreeBSD futex emulation, context switching, fd handling - bvar/default_variables.cpp: FreeBSD /proc fallback Most changes piggyback on existing macOS/Darwin code paths since both use kqueue. The NO_PTHREAD_MUTEX_HOOK fix is critical — without it, any FreeBSD application linking brpc will deadlock during startup.
1 parent 5a2466a commit c836aff

22 files changed

Lines changed: 170 additions & 37 deletions

‎CMakeLists.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,11 @@ endif()
147147
set(CMAKE_CPP_FLAGS "${CMAKE_CPP_FLAGS} -DBTHREAD_USE_FAST_PTHREAD_MUTEX -D__const__=__unused__ -D_GNU_SOURCE -DUSE_SYMBOLIZE -DNO_TCMALLOC -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -DBRPC_REVISION=\\\"${BRPC_REVISION}\\\" -D__STRICT_ANSI__")
148148
set(CMAKE_CPP_FLAGS "${CMAKE_CPP_FLAGS} ${DEBUG_SYMBOL} ${THRIFT_CPP_FLAG}")
149149
set(CMAKE_CXX_FLAGS "${CMAKE_CPP_FLAGS} ${CMAKE_CXX_FLAGS} -O2 -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer")
150+
151+
if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
152+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNO_PTHREAD_MUTEX_HOOK")
153+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DNO_PTHREAD_MUTEX_HOOK")
154+
endif()
150155
set(CMAKE_C_FLAGS "${CMAKE_CPP_FLAGS} -O2 -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-unused-parameter -fno-omit-frame-pointer")
151156

152157
macro(use_cxx11)
@@ -342,6 +347,9 @@ endif()
342347
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
343348
set(DYNAMIC_LIB ${DYNAMIC_LIB} rt)
344349
set(BRPC_PRIVATE_LIBS "${BRPC_PRIVATE_LIBS} -lrt")
350+
elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
351+
set(DYNAMIC_LIB ${DYNAMIC_LIB} execinfo)
352+
set(BRPC_PRIVATE_LIBS "${BRPC_PRIVATE_LIBS} -lexecinfo")
345353
elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
346354
set(DYNAMIC_LIB ${DYNAMIC_LIB}
347355
pthread
@@ -500,6 +508,10 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
500508
${PROJECT_SOURCE_DIR}/src/butil/strings/sys_string_conversions_mac.mm
501509
${PROJECT_SOURCE_DIR}/src/butil/time/time_mac.cc
502510
${PROJECT_SOURCE_DIR}/src/butil/mac/scoped_mach_port.cc)
511+
elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
512+
set(BUTIL_SOURCES ${BUTIL_SOURCES}
513+
${PROJECT_SOURCE_DIR}/src/butil/threading/platform_thread_freebsd.cc
514+
${PROJECT_SOURCE_DIR}/src/butil/strings/sys_string_conversions_posix.cc)
503515
endif()
504516

505517
file(GLOB_RECURSE BVAR_SOURCES "${PROJECT_SOURCE_DIR}/src/bvar/*.cpp")

‎src/brpc/controller.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1578,7 +1578,7 @@ void Controller::CallAfterRpcResp(const google::protobuf::Message* req, const go
15781578
}
15791579
}
15801580

1581-
#if defined(OS_MACOSX)
1581+
#if defined(OS_MACOSX) || defined(OS_FREEBSD)
15821582
typedef sig_t SignalHandler;
15831583
#else
15841584
typedef sighandler_t SignalHandler;

‎src/brpc/event_dispatcher.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ void IOEventData::BeforeRecycled() {
101101

102102
#if defined(OS_LINUX)
103103
#include "brpc/event_dispatcher_epoll.cpp"
104-
#elif defined(OS_MACOSX)
104+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
105105
#include "brpc/event_dispatcher_kqueue.cpp"
106106
#else
107107
#error Not implemented

‎src/brpc/policy/domain_naming_service.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ int DomainNamingService::GetServers(const char* dns_name,
106106

107107
}
108108

109-
#if defined(OS_MACOSX)
109+
#if defined(OS_MACOSX) || defined(OS_FREEBSD)
110110
_aux_buf_len = 0; // suppress unused warning
111111
// gethostbyname on MAC is thread-safe (with current usage) since the
112112
// returned hostent is TLS. Check following link for the ref:

‎src/brpc/socket.cpp‎

Lines changed: 53 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,9 @@
5151
#include "brpc/periodic_task.h"
5252
#include "brpc/details/health_check.h"
5353
#include "brpc/transport_factory.h"
54-
#if defined(OS_MACOSX)
54+
#include "brpc/rdma/rdma_endpoint.h"
55+
#include "brpc/rdma/rdma_helper.h"
56+
#if defined(OS_MACOSX) || defined(OS_FREEBSD)
5557
#include <sys/event.h>
5658
#endif
5759

@@ -676,6 +678,30 @@ void Socket::SetSocketOptions(int fd) {
676678
PLOG(ERROR) << "Fail to set keepcnt of fd=" << fd;
677679
}
678680
}
681+
#elif defined(OS_FREEBSD)
682+
if (_keepalive_options->keepalive_idle_s > 0) {
683+
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE,
684+
&_keepalive_options->keepalive_idle_s,
685+
sizeof(_keepalive_options->keepalive_idle_s)) != 0) {
686+
PLOG(ERROR) << "Fail to set keepidle of fd=" << fd;
687+
}
688+
}
689+
690+
if (_keepalive_options->keepalive_interval_s > 0) {
691+
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL,
692+
&_keepalive_options->keepalive_interval_s,
693+
sizeof(_keepalive_options->keepalive_interval_s)) != 0) {
694+
PLOG(ERROR) << "Fail to set keepintvl of fd=" << fd;
695+
}
696+
}
697+
698+
if (_keepalive_options->keepalive_count > 0) {
699+
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT,
700+
&_keepalive_options->keepalive_count,
701+
sizeof(_keepalive_options->keepalive_count)) != 0) {
702+
PLOG(ERROR) << "Fail to set keepcnt of fd=" << fd;
703+
}
704+
}
679705
#elif defined(OS_MACOSX)
680706
if (_keepalive_options->keepalive_idle_s > 0) {
681707
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE,
@@ -2006,7 +2032,7 @@ int Socket::SSLHandshake(int fd, bool server_mode) {
20062032
case SSL_ERROR_WANT_READ:
20072033
#if defined(OS_LINUX)
20082034
if (bthread_fd_wait(fd, EPOLLIN) != 0) {
2009-
#elif defined(OS_MACOSX)
2035+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
20102036
if (bthread_fd_wait(fd, EVFILT_READ) != 0) {
20112037
#endif
20122038
return -1;
@@ -2016,7 +2042,7 @@ int Socket::SSLHandshake(int fd, bool server_mode) {
20162042
case SSL_ERROR_WANT_WRITE:
20172043
#if defined(OS_LINUX)
20182044
if (bthread_fd_wait(fd, EPOLLOUT) != 0) {
2019-
#elif defined(OS_MACOSX)
2045+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
20202046
if (bthread_fd_wait(fd, EVFILT_WRITE) != 0) {
20212047
#endif
20222048
return -1;
@@ -2183,7 +2209,7 @@ int Socket::OnInputEvent(void* user_data, uint32_t events,
21832209
if (s->fd() < 0) {
21842210
#if defined(OS_LINUX)
21852211
CHECK(!(events & EPOLLIN)) << "epoll_events=" << events;
2186-
#elif defined(OS_MACOSX)
2212+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
21872213
CHECK((short)events != EVFILT_READ) << "kqueue filter=" << events;
21882214
#endif
21892215
return -1;
@@ -2406,7 +2432,11 @@ void Socket::DebugSocket(std::ostream& os, SocketId id) {
24062432
{
24072433
int keepidle = 0;
24082434
socklen_t len = sizeof(keepidle);
2409-
#if defined(OS_MACOSX)
2435+
#if defined(OS_FREEBSD)
2436+
if (getsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepidle, &len) == 0) {
2437+
os << "\ntcp_keepalive_time=" << keepidle;
2438+
}
2439+
#elif defined(OS_MACOSX)
24102440
if (getsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &keepidle, &len) == 0) {
24112441
os << "\ntcp_keepalive_time=" << keepidle;
24122442
}
@@ -2420,7 +2450,7 @@ void Socket::DebugSocket(std::ostream& os, SocketId id) {
24202450
{
24212451
int keepintvl = 0;
24222452
socklen_t len = sizeof(keepintvl);
2423-
#if defined(OS_MACOSX)
2453+
#if defined(OS_MACOSX) || defined(OS_FREEBSD)
24242454
if (getsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &keepintvl, &len) == 0) {
24252455
os << "\ntcp_keepalive_intvl=" << keepintvl;
24262456
}
@@ -2434,7 +2464,7 @@ void Socket::DebugSocket(std::ostream& os, SocketId id) {
24342464
{
24352465
int keepcnt = 0;
24362466
socklen_t len = sizeof(keepcnt);
2437-
#if defined(OS_MACOSX)
2467+
#if defined(OS_MACOSX) || defined(OS_FREEBSD)
24382468
if (getsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &keepcnt, &len) == 0) {
24392469
os << "\ntcp_keepalive_probes=" << keepcnt;
24402470
}
@@ -2455,7 +2485,22 @@ void Socket::DebugSocket(std::ostream& os, SocketId id) {
24552485
}
24562486
#endif
24572487

2458-
#if defined(OS_MACOSX)
2488+
#if defined(OS_FREEBSD)
2489+
struct tcp_info ti;
2490+
socklen_t len = sizeof(ti);
2491+
if (fd >= 0 && getsockopt(fd, IPPROTO_TCP, TCP_INFO, &ti, &len) == 0) {
2492+
os << "\ntcpi={\n state=" << (uint32_t)ti.tcpi_state
2493+
<< "\n snd_wscale=" << (uint32_t)ti.tcpi_snd_wscale
2494+
<< "\n rcv_wscale=" << (uint32_t)ti.tcpi_rcv_wscale
2495+
<< "\n options=" << (uint32_t)ti.tcpi_options
2496+
<< "\n rto=" << ti.tcpi_rto
2497+
<< "\n snd_mss=" << ti.tcpi_snd_mss
2498+
<< "\n snd_ssthresh=" << ti.tcpi_snd_ssthresh
2499+
<< "\n snd_cwnd=" << ti.tcpi_snd_cwnd
2500+
<< "\n rcv_space=" << ti.tcpi_rcv_space
2501+
<< "\n}";
2502+
}
2503+
#elif defined(OS_MACOSX)
24592504
struct tcp_connection_info ti;
24602505
socklen_t len = sizeof(ti);
24612506
if (fd >= 0 && getsockopt(fd, IPPROTO_TCP, TCP_CONNECTION_INFO, &ti, &len) == 0) {

‎src/bthread/context.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,15 @@
5858
#define BTHREAD_CONTEXT_CALL_CONVENTION __cdecl
5959
#endif
6060

61+
#elif defined(__FreeBSD__)
62+
#ifdef __x86_64__
63+
#define BTHREAD_CONTEXT_PLATFORM_linux_x86_64
64+
#define BTHREAD_CONTEXT_CALL_CONVENTION
65+
#elif __aarch64__
66+
#define BTHREAD_CONTEXT_PLATFORM_linux_arm64
67+
#define BTHREAD_CONTEXT_CALL_CONVENTION
68+
#endif
69+
6170
#elif defined(__APPLE__) && defined(__MACH__)
6271
#if defined (__i386__)
6372
#define BTHREAD_CONTEXT_PLATFORM_apple_i386

‎src/bthread/errno.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ extern int *__errno_location() __attribute__((__const__));
3535
int *bthread_errno_location() {
3636
return __errno_location();
3737
}
38-
#elif defined(OS_MACOSX)
38+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
3939

4040
extern int * __error(void);
4141

‎src/bthread/fd.cpp‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
#include "butil/compat.h"
2323
#include <new> // std::nothrow
2424
#include <sys/poll.h> // poll()
25-
#if defined(OS_MACOSX)
25+
#if defined(OS_MACOSX) || defined(OS_FREEBSD)
2626
#include <sys/types.h> // struct kevent
2727
#include <sys/event.h> // kevent(), kqueue()
2828
#endif
@@ -133,7 +133,7 @@ class EpollThread {
133133
}
134134
#if defined(OS_LINUX)
135135
_epfd = epoll_create(epoll_size);
136-
#elif defined(OS_MACOSX)
136+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
137137
_epfd = kqueue();
138138
#endif
139139
_start_mutex.unlock();
@@ -181,7 +181,7 @@ class EpollThread {
181181
epoll_event evt = { EPOLLOUT, { NULL } };
182182
if (epoll_ctl(saved_epfd, EPOLL_CTL_ADD,
183183
closing_epoll_pipe[1], &evt) < 0) {
184-
#elif defined(OS_MACOSX)
184+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
185185
struct kevent kqueue_event;
186186
EV_SET(&kqueue_event, closing_epoll_pipe[1], EVFILT_WRITE, EV_ADD | EV_ENABLE,
187187
0, 0, NULL);
@@ -257,7 +257,7 @@ class EpollThread {
257257
return -1;
258258
}
259259
# endif
260-
#elif defined(OS_MACOSX)
260+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
261261
struct kevent kqueue_event;
262262
EV_SET(&kqueue_event, fd, events, EV_ADD | EV_ENABLE | EV_ONESHOT,
263263
0, 0, butex);
@@ -299,7 +299,7 @@ class EpollThread {
299299
}
300300
#if defined(OS_LINUX)
301301
epoll_ctl(_epfd, EPOLL_CTL_DEL, fd, NULL);
302-
#elif defined(OS_MACOSX)
302+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
303303
struct kevent evt;
304304
EV_SET(&evt, fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
305305
kevent(_epfd, &evt, 1, NULL, 0, NULL);
@@ -325,7 +325,7 @@ class EpollThread {
325325
const size_t MAX_EVENTS = 32;
326326
#if defined(OS_LINUX)
327327
epoll_event* e = new (std::nothrow) epoll_event[MAX_EVENTS];
328-
#elif defined(OS_MACOSX)
328+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
329329
typedef struct kevent KEVENT;
330330
struct kevent* e = new (std::nothrow) KEVENT[MAX_EVENTS];
331331
#endif
@@ -343,7 +343,7 @@ class EpollThread {
343343
const int epfd = _epfd;
344344
#if defined(OS_LINUX)
345345
const int n = epoll_wait(epfd, e, MAX_EVENTS, -1);
346-
#elif defined(OS_MACOSX)
346+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
347347
const int n = kevent(epfd, NULL, 0, e, MAX_EVENTS, NULL);
348348
#endif
349349
if (_stop) {
@@ -383,7 +383,7 @@ class EpollThread {
383383
EpollButex* butex = pbutex ?
384384
pbutex->load(butil::memory_order_consume) : NULL;
385385
# endif
386-
#elif defined(OS_MACOSX)
386+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
387387
EpollButex* butex = static_cast<EpollButex*>(e[i].udata);
388388
#endif
389389
if (butex != NULL && butex != CLOSING_GUARD) {
@@ -495,7 +495,7 @@ int bthread_connect(int sockfd, const sockaddr* serv_addr,
495495
}
496496
#if defined(OS_LINUX)
497497
if (bthread_fd_wait(sockfd, EPOLLOUT) < 0) {
498-
#elif defined(OS_MACOSX)
498+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
499499
if (bthread_fd_wait(sockfd, EVFILT_WRITE) < 0) {
500500
#endif
501501
return -1;
@@ -531,7 +531,7 @@ int bthread_timed_connect(int sockfd, const struct sockaddr* serv_addr,
531531
}
532532
#if defined(OS_LINUX)
533533
if (bthread_fd_timedwait(sockfd, EPOLLOUT, abstime) < 0) {
534-
#elif defined(OS_MACOSX)
534+
#elif defined(OS_MACOSX) || defined(OS_FREEBSD)
535535
if (bthread_fd_timedwait(sockfd, EVFILT_WRITE, abstime) < 0) {
536536
#endif
537537
return -1;

‎src/bthread/mutex.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -477,6 +477,12 @@ static void init_sys_mutex_lock() {
477477
sys_pthread_mutex_lock = (MutexOp)dlsym(RTLD_NEXT, "pthread_mutex_lock");
478478
sys_pthread_mutex_trylock = (MutexOp)dlsym(RTLD_NEXT, "pthread_mutex_trylock");
479479
sys_pthread_mutex_unlock = (MutexOp)dlsym(RTLD_NEXT, "pthread_mutex_unlock");
480+
#elif defined(OS_FREEBSD)
481+
sys_pthread_mutex_init = (MutexInitOp)dlsym(RTLD_NEXT, "pthread_mutex_init");
482+
sys_pthread_mutex_destroy = (MutexOp)dlsym(RTLD_NEXT, "pthread_mutex_destroy");
483+
sys_pthread_mutex_lock = (MutexOp)dlsym(RTLD_NEXT, "pthread_mutex_lock");
484+
sys_pthread_mutex_trylock = (MutexOp)dlsym(RTLD_NEXT, "pthread_mutex_trylock");
485+
sys_pthread_mutex_unlock = (MutexOp)dlsym(RTLD_NEXT, "pthread_mutex_unlock");
480486
#endif
481487
}
482488

‎src/bthread/sys_futex.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
#include <pthread.h>
2626
#include <unordered_map>
2727

28-
#if defined(OS_MACOSX)
28+
#if defined(OS_MACOSX) || defined(OS_FREEBSD)
2929

3030
namespace bthread {
3131

0 commit comments

Comments
 (0)