BUG/MEDIUM: block_secrets stability regression

last change expanded the LD_PRELOAD interposer from the open() family to
~50 libc calls, including the stat/lstat/fstatat/statx/access/faccessat/
readlink/readlinkat/opendir family that HAProxy exercises constantly on its
hot path, including inside process_chk while holding the per-SERVER lock. Each
intercepted call runs is_blocked(), whose slow path executes a blocking,
non-async-signal-safe realpath(); doing that in the event loop stalls threads
on the SERVER lock and trips the "stopped processing traffic" watchdog, which
escalates to recovery/crash (issue #818).

Narrow interception back to the calls that actually read or mutate secrets
(open/openat/fopen/freopen/creat and the write hooks) and drop the metadata-
only hooks, so HAProxy's stat/access/readlink bind straight to libc and never
detour through realpath().

Also fix a related musl bug: the LFS64 aliases (open64/stat64/fstatat64/
truncate64/renameat2) are not exported by musl, so dlsym(RTLD_NEXT, ...)
returned NULL and the exported *64 hooks returned ENOSYS while HOOK_GUARD
re-dlsym()'d on every call. Alias the *64 pointers to their base counterparts
(ABI-identical on 64-bit) and special-case renameat2(flags==0) -> renameat.

Add behavioural tests (block_secrets_test.{c,sh}) covering the security
invariant and both regressions, runnable under the musl/alpine build image.

Author: dkorunic

.aspell.yml

@@ -123,3 +123,17 @@ allowed:
   - GVK
   - Impl
   - failfast
+  - PEM
+  - redact
+  - httpresponses
+  - chk
+  - faccessat
+  - fstatat
+  - interposer
+  - lstat
+  - opendir
+  - readlink
+  - readlinkat
+  - realpath
+  - runnable
+  - statx

hug/protection/block_secrets.c

@@ -12,9 +12,14 @@
 /* See the License for the specific language governing permissions and        */
 /* limitations under the License.                                             */
 
+/* LD_PRELOAD shim that denies HAProxy access to the Kubernetes service-account
+   token dir (default /var/run/secrets/kubernetes.io). Interposes the libc
+   calls that read or mutate a path and returns EACCES for anything resolving
+   inside it. Metadata-only calls (stat/access/readlink) are deliberately NOT
+   hooked so they stay off HAProxy's hot path (issue #818). */
+
 #define _XOPEN_SOURCE 700
 
-#include <dirent.h>
 #include <dlfcn.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -34,7 +39,6 @@
 /* Forward-declared to avoid _GNU_SOURCE, which on musl macro-aliases the
    *64 stat names and would collide with our hook symbols. */
 struct file_handle;
-struct statx;
 
 #ifndef PATH_MAX
 #define PATH_MAX 4096
@@ -54,7 +58,7 @@ static char text_blocked[PATH_MAX] = {0};
 static size_t text_blocked_len = 0;
 static int text_canonical_same = 1;
 
-/* Recursion guard: musl's realpath() uses the public readlink symbol
+/* Recursion guard: realpath() resolves paths via the public open() symbol,
    which our hook would otherwise re-enter. */
 static __thread int in_is_blocked = 0;
 
@@ -218,21 +222,6 @@ static int (*real_creat)(const char *, mode_t) = NULL;
 static int (*real_creat64)(const char *, mode_t) = NULL;
 static int (*real_openat)(int, const char *, int, ...) = NULL;
 static int (*real_openat64)(int, const char *, int, ...) = NULL;
-static int (*real_access)(const char *, int) = NULL;
-static int (*real_faccessat)(int, const char *, int, int) = NULL;
-static ssize_t (*real_readlink)(const char *, char *, size_t) = NULL;
-static ssize_t (*real_readlinkat)(int, const char *, char *, size_t) = NULL;
-static DIR *(*real_opendir)(const char *) = NULL;
-static int (*real_stat)(const char *, struct stat *) = NULL;
-static int (*real_lstat)(const char *, struct stat *) = NULL;
-static int (*real_fstatat)(int, const char *, struct stat *, int) = NULL;
-/* void * avoids _LARGEFILE64_SOURCE, which on musl macro-aliases stat64
-   to stat and would break our hook symbol. */
-static int (*real_stat64)(const char *, void *) = NULL;
-static int (*real_lstat64)(const char *, void *) = NULL;
-static int (*real_fstatat64)(int, const char *, void *, int) = NULL;
-static int (*real_statx)(int, const char *, int, unsigned int,
-                         struct statx *) = NULL;
 static int (*real_name_to_handle_at)(int, const char *, struct file_handle *,
                                      int *, int) = NULL;
 static int (*real_mkdir)(const char *, mode_t) = NULL;
@@ -276,18 +265,6 @@ __attribute__((constructor(101), cold)) static void block_secrets_init(void) {
   real_creat64 = dlsym(RTLD_NEXT, "creat64");
   real_openat = dlsym(RTLD_NEXT, "openat");
   real_openat64 = dlsym(RTLD_NEXT, "openat64");
-  real_access = dlsym(RTLD_NEXT, "access");
-  real_faccessat = dlsym(RTLD_NEXT, "faccessat");
-  real_readlink = dlsym(RTLD_NEXT, "readlink");
-  real_readlinkat = dlsym(RTLD_NEXT, "readlinkat");
-  real_opendir = dlsym(RTLD_NEXT, "opendir");
-  real_stat = dlsym(RTLD_NEXT, "stat");
-  real_lstat = dlsym(RTLD_NEXT, "lstat");
-  real_fstatat = dlsym(RTLD_NEXT, "fstatat");
-  real_stat64 = dlsym(RTLD_NEXT, "stat64");
-  real_lstat64 = dlsym(RTLD_NEXT, "lstat64");
-  real_fstatat64 = dlsym(RTLD_NEXT, "fstatat64");
-  real_statx = dlsym(RTLD_NEXT, "statx");
   real_name_to_handle_at = dlsym(RTLD_NEXT, "name_to_handle_at");
   real_mkdir = dlsym(RTLD_NEXT, "mkdir");
   real_mkdirat = dlsym(RTLD_NEXT, "mkdirat");
@@ -314,6 +291,28 @@ __attribute__((constructor(101), cold)) static void block_secrets_init(void) {
   real_symlink = dlsym(RTLD_NEXT, "symlink");
   real_symlinkat = dlsym(RTLD_NEXT, "symlinkat");
 
+  /* musl exports no distinct LFS64 symbols, so the dlsym()s above leave these
+     NULL. off_t is 64-bit on every supported ABI, so *64 == base: alias them.
+     Otherwise the exported *64 hooks would return ENOSYS (issue #818). */
+  if (real_open64 == NULL) {
+    real_open64 = real_open;
+  }
+  if (real_fopen64 == NULL) {
+    real_fopen64 = real_fopen;
+  }
+  if (real_freopen64 == NULL) {
+    real_freopen64 = real_freopen;
+  }
+  if (real_creat64 == NULL) {
+    real_creat64 = real_creat;
+  }
+  if (real_openat64 == NULL) {
+    real_openat64 = real_openat;
+  }
+  if (real_truncate64 == NULL) {
+    real_truncate64 = (int (*)(const char *, int64_t))real_truncate;
+  }
+
   const char *src = getenv(BLOCKED_PATH_ENV);
   if (src == NULL) {
     src = BLOCKED_PATH_DEFAULT;
@@ -477,119 +476,6 @@ HOOK_VISIBLE int openat64(int dirfd, const char *pathname, int flags, ...) {
   return real_openat64(dirfd, pathname, flags, mode);
 }
 
-HOOK_VISIBLE int access(const char *pathname, int mode) {
-  if (path_blocked(pathname)) {
-    errno = EACCES;
-    return -1;
-  }
-  HOOK_GUARD(real_access, -1);
-  return real_access(pathname, mode);
-}
-
-HOOK_VISIBLE int faccessat(int dirfd, const char *pathname, int mode,
-                           int flags) {
-  if (path_blocked(pathname)) {
-    errno = EACCES;
-    return -1;
-  }
-  HOOK_GUARD(real_faccessat, -1);
-  return real_faccessat(dirfd, pathname, mode, flags);
-}
-
-HOOK_VISIBLE ssize_t readlink(const char *pathname, char *buf, size_t bufsiz) {
-  if (path_blocked(pathname)) {
-    errno = EACCES;
-    return -1;
-  }
-  HOOK_GUARD(real_readlink, -1);
-  return real_readlink(pathname, buf, bufsiz);
-}
-
-HOOK_VISIBLE ssize_t readlinkat(int dirfd, const char *pathname, char *buf,
-                                size_t bufsiz) {
-  if (path_blocked(pathname)) {
-    errno = EACCES;
-    return -1;
-  }
-  HOOK_GUARD(real_readlinkat, -1);
-  return real_readlinkat(dirfd, pathname, buf, bufsiz);
-}
-
-HOOK_VISIBLE DIR *opendir(const char *name) {
-  if (path_blocked(name)) {
-    errno = EACCES;
-    return NULL;
-  }
-  HOOK_GUARD(real_opendir, NULL);
-  return real_opendir(name);
-}
-
-HOOK_VISIBLE int stat(const char *pathname, struct stat *buf) {
-  if (path_blocked(pathname)) {
-    errno = EACCES;
-    return -1;
-  }
-  HOOK_GUARD(real_stat, -1);
-  return real_stat(pathname, buf);
-}
-
-HOOK_VISIBLE int lstat(const char *pathname, struct stat *buf) {
-  if (path_blocked(pathname)) {
-    errno = EACCES;
-    return -1;
-  }
-  HOOK_GUARD(real_lstat, -1);
-  return real_lstat(pathname, buf);
-}
-
-HOOK_VISIBLE int fstatat(int dirfd, const char *pathname, struct stat *buf,
-                         int flags) {
-  if (path_blocked(pathname)) {
-    errno = EACCES;
-    return -1;
-  }
-  HOOK_GUARD(real_fstatat, -1);
-  return real_fstatat(dirfd, pathname, buf, flags);
-}
-
-HOOK_VISIBLE int stat64(const char *pathname, void *buf) {
-  if (path_blocked(pathname)) {
-    errno = EACCES;
-    return -1;
-  }
-  HOOK_GUARD(real_stat64, -1);
-  return real_stat64(pathname, buf);
-}
-
-HOOK_VISIBLE int lstat64(const char *pathname, void *buf) {
-  if (path_blocked(pathname)) {
-    errno = EACCES;
-    return -1;
-  }
-  HOOK_GUARD(real_lstat64, -1);
-  return real_lstat64(pathname, buf);
-}
-
-HOOK_VISIBLE int fstatat64(int dirfd, const char *pathname, void *buf,
-                           int flags) {
-  if (path_blocked(pathname)) {
-    errno = EACCES;
-    return -1;
-  }
-  HOOK_GUARD(real_fstatat64, -1);
-  return real_fstatat64(dirfd, pathname, buf, flags);
-}
-
-HOOK_VISIBLE int statx(int dirfd, const char *pathname, int flags,
-                       unsigned int mask, struct statx *buf) {
-  if (path_blocked(pathname)) {
-    errno = EACCES;
-    return -1;
-  }
-  HOOK_GUARD(real_statx, -1);
-  return real_statx(dirfd, pathname, flags, mask, buf);
-}
-
 HOOK_VISIBLE int name_to_handle_at(int dirfd, const char *pathname,
                                    struct file_handle *handle, int *mount_id,
                                    int flags) {
@@ -783,7 +669,15 @@ HOOK_VISIBLE int renameat2(int olddirfd, const char *oldpath, int newdirfd,
     errno = EACCES;
     return -1;
   }
-  HOOK_GUARD(real_renameat2, -1);
+  /* No renameat2 on musl: flags==0 is plain renameat(); a non-zero flag is
+     genuinely unsupported, so ENOSYS is then the correct answer. */
+  if (real_renameat2 == NULL) {
+    if (flags == 0 && real_renameat != NULL) {
+      return real_renameat(olddirfd, oldpath, newdirfd, newpath);
+    }
+    errno = ENOSYS;
+    return -1;
+  }
   return real_renameat2(olddirfd, oldpath, newdirfd, newpath, flags);
 }
 

hug/protection/block_secrets_test.c

@@ -0,0 +1,91 @@
+/* Copyright 2025 HAProxy Technologies LLC                                    */
+/*                                                                            */
+/* Licensed under the Apache License, Version 2.0 (the "License");            */
+/* you may not use this file except in compliance with the License.           */
+/* You may obtain a copy of the License at                                    */
+/*                                                                            */
+/*    http://www.apache.org/licenses/LICENSE-2.0                              */
+/*                                                                            */
+/* Unless required by applicable law or agreed to in writing, software        */
+/* distributed under the License is distributed on an "AS IS" BASIS,          */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   */
+/* See the License for the specific language governing permissions and        */
+/* limitations under the License.                                             */
+
+/* Behavioural tests for libblock_secrets.so, run preloaded by
+   block_secrets_test.sh. Regression coverage for issue #818:
+     - content reads (open/fopen) into the protected dir stay blocked;
+     - metadata calls (stat/access) are no longer hooked;
+     - LFS64 aliases (open64) must not return ENOSYS on musl. */
+
+#define _GNU_SOURCE
+#include <dlfcn.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+static int failures = 0;
+
+static void check(int cond, const char *what) {
+  printf("  [%s] %s\n", cond ? "PASS" : "FAIL", what);
+  if (!cond) {
+    failures++;
+  }
+}
+
+int main(void) {
+  const char *blocked_file = getenv("TEST_BLOCKED_FILE"); /* inside protected dir */
+  const char *normal_file = getenv("TEST_NORMAL_FILE");   /* outside protected dir */
+  struct stat st;
+  int fd;
+  FILE *fp;
+
+  printf("== security: content reads into the protected dir stay blocked ==\n");
+  errno = 0;
+  fd = open(blocked_file, O_RDONLY);
+  check(fd == -1 && errno == EACCES, "open(blocked) is denied with EACCES");
+  if (fd != -1) {
+    close(fd);
+  }
+  errno = 0;
+  fp = fopen(blocked_file, "r");
+  check(fp == NULL && errno == EACCES, "fopen(blocked) is denied with EACCES");
+  if (fp) {
+    fclose(fp);
+  }
+
+  printf("== regression #818: metadata-only calls are no longer intercepted ==\n");
+  errno = 0;
+  check(stat(blocked_file, &st) == 0, "stat(blocked) succeeds (not hooked)");
+  errno = 0;
+  check(access(blocked_file, F_OK) == 0, "access(blocked) succeeds (not hooked)");
+
+  printf("== regression #818: LFS64 aliases must not return ENOSYS ==\n");
+  int (*p_open64)(const char *, int, ...) = (void *)dlsym(RTLD_DEFAULT, "open64");
+  if (p_open64) {
+    errno = 0;
+    fd = p_open64(normal_file, O_RDONLY);
+    check(fd >= 0, "open64(normal) succeeds (no ENOSYS)");
+    if (fd >= 0) {
+      close(fd);
+    }
+  } else {
+    printf("  [SKIP] open64 symbol absent\n");
+  }
+
+  printf("== sanity: normal files remain accessible ==\n");
+  errno = 0;
+  fd = open(normal_file, O_RDONLY);
+  check(fd >= 0, "open(normal) succeeds");
+  if (fd >= 0) {
+    close(fd);
+  }
+
+  printf("\n%s (%d failure(s))\n", failures ? "TESTS FAILED" : "TESTS PASSED",
+         failures);
+  return failures ? 1 : 0;
+}

hug/protection/block_secrets_test.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+# Copyright 2025 HAProxy Technologies LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+#
+# Builds libblock_secrets.so and runs block_secrets_test.c under LD_PRELOAD.
+# Designed to run inside the musl/alpine build image so the test exercises the
+# same libc as production. Usage:
+#   docker run --rm -v "$PWD/pkg/protection":/p:ro haproxytech/haproxy-alpine:3.2 \
+#       sh -c 'apk add --no-cache build-base >/dev/null && cp /p/* /tmp && /tmp/block_secrets_test.sh'
+set -eu
+
+SRC_DIR="$(cd "$(dirname "$0")" && pwd)"
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+
+cc -O2 -std=c11 -fPIC -shared -o "$WORK/libblock_secrets.so" \
+    "$SRC_DIR/block_secrets.c" -ldl
+cc -O2 -std=c11 -o "$WORK/block_secrets_test" "$SRC_DIR/block_secrets_test.c" -ldl
+
+# Protected dir with a "token" inside, plus a normal file outside it.
+BLOCKED_DIR="$WORK/secrets/kubernetes.io"
+mkdir -p "$BLOCKED_DIR"
+echo "secret-token" > "$BLOCKED_DIR/token"
+echo "data" > "$WORK/normal.txt"
+
+BLOCK_SECRETS_PATH="$BLOCKED_DIR" \
+TEST_BLOCKED_FILE="$BLOCKED_DIR/token" \
+TEST_NORMAL_FILE="$WORK/normal.txt" \
+LD_PRELOAD="$WORK/libblock_secrets.so" \
+    "$WORK/block_secrets_test"

hug/protection/haproxy_wrapper.c

@@ -12,6 +12,10 @@
 /* See the License for the specific language governing permissions and        */
 /* limitations under the License.                                             */
 
+/* Thin entrypoint that forces LD_PRELOAD=libblock_secrets.so and execs the
+   real HAProxy, so the secrets-blocker is always loaded regardless of how the
+   process is started. The Dockerfile grants this binary cap_net_bind_service. */
+
 #define _GNU_SOURCE
 #include <errno.h>
 #include <stdio.h>