Test old Linux kernels using Qemu (#280)

cmcgee1024 · web-flow · commit 6b0d119fa7a0 · 2026-06-05T18:58:13.000-04:00
Fixes the following:

* Increases test confidence with older Linux kernels (4.18, and 5.10)
* Closes file handles that were leaking on Linux
* Better soft limit detection so that testConcurrentRun() can avoid exhausting system open file limit
* Handle the epoll(DEL) case when a process is being shut down for Linux with older kernels
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -58,6 +58,22 @@ jobs:
         done
         # empty line to ignore the --swift-sdk given by swiftlang/github-workflows/.github/workflows/scripts/install-and-build-with-sdk.sh \
 
+  test_linux_kernel:
+    name: Test Linux Kernel / ${{ matrix.dist-kern }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # These are specific distro and kernel versions that the test qemu script supports from here: https://images.linuxcontainers.org
+        dist-kern: ["al2-4.18", "al2-5.10"]
+    container:
+      image: ubuntu:24.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Run Test
+        run: bash -c './scripts/test-using-qemu.sh ${{ matrix.dist-kern }} -- swift test'
+
   soundness:
     name: Soundness
     uses: swiftlang/github-workflows/.github/workflows/soundness.yml@0.0.11
diff --git a/.swift-version b/.swift-version
@@ -1 +1 @@
-6.2.0
+6.3.2
diff --git a/Sources/Subprocess/Platforms/Subprocess+Linux.swift b/Sources/Subprocess/Platforms/Subprocess+Linux.swift
@@ -447,17 +447,37 @@ private func _unregisterProcessDescriptorAndNotify(_ pidfd: CInt, context: Monit
         newStorage.continuations.removeValue(forKey: pidfd)
         state = .started(newStorage)
 
-        // Remove this pidfd from epoll to prevent further notifications
-        let rc = epoll_ctl(
+        // Remove this pidfd from epoll to prevent further notifications.
+        // The return value is intentionally not propagated to the continuation:
+        // epoll firing this event means the process has already exited, so
+        // monitoring succeeded regardless of cleanup outcome.
+        //
+        // ENOENT is silently ignored: it means the fd is not (or is no longer)
+        // in the epoll instance, which is harmless — this occurs on concurrent
+        // removals and on older 5.x kernels where epoll_ctl(DEL) incorrectly
+        // reports ENOENT for pidfds after process exit.  The fd is removed from
+        // epoll automatically by the kernel when processIdentifier.close() closes
+        // it anyway, so a failed DEL is never permanent.
+        //
+        // Any other error (EBADF, EINVAL, …) would indicate a programming error
+        // in fd lifecycle management — e.g. the pidfd was closed prematurely —
+        // and is surfaced as an assertion failure in debug builds.
+        let delRC = epoll_ctl(
             context.epollFileDescriptor,
             EPOLL_CTL_DEL,
             pidfd,
             nil
         )
-        if rc != 0 {
-            let epollErrno = errno
+
+        // The pidfd is intentionally left open here.  It is owned by
+        // ProcessIdentifier and will be closed by processIdentifier.close()
+        // in the defer in Configuration.swift once monitoring is fully done.
+        // Closing it here would free the fd number and allow it to be recycled
+        // before that defer runs, causing a close-the-wrong-fd race.
+
+        if delRC != 0 && errno != ENOENT {
             let error = SubprocessError.failedToMonitor(
-                withUnderlyingError: Errno(rawValue: epollErrno)
+                withUnderlyingError: Errno(rawValue: errno)
             )
             return (continuationList, error)
         }
@@ -552,6 +572,7 @@ internal func _isWaitprocessDescriptorSupported() -> Bool {
         // If we can not retrieve pidfd, the system does not support waitid(P_PIDFD)
         return false
     }
+    defer { try? FileDescriptor(rawValue: selfPidfd).close() }
     /// The following call will fail either with
     /// - ECHILD: in this case we know P_PIDFD is supported and waitid correctly
     ///     reported that we don't have a child with the same selfPidfd;
diff --git a/Sources/Subprocess/Platforms/Subprocess+Unix.swift b/Sources/Subprocess/Platforms/Subprocess+Unix.swift
@@ -533,6 +533,16 @@ extension Configuration {
                 // Spawn error
                 if spawnError != 0 {
                     if [ENOENT, EACCES, ENOTDIR].contains(spawnError) {
+                        // clone3(CLONE_PIDFD) allocates a pidfd before exec runs.
+                        // If exec fails we retry with the next candidate path, so
+                        // close the pidfd here to avoid leaking it across retries.
+                        if processDescriptor != .invalidDescriptor {
+                            do {
+                                try FileDescriptor(rawValue: processDescriptor).close()
+                            } catch {
+                                throw SubprocessError.spawnFailed(withUnderlyingError: error as? SubprocessError.UnderlyingError)
+                            }
+                        }
                         // Move on to another possible path
                         continue
                     }
diff --git a/Sources/_SubprocessCShims/include/process_shims.h b/Sources/_SubprocessCShims/include/process_shims.h
@@ -17,6 +17,7 @@
 #if !TARGET_OS_WINDOWS
 #include <pthread.h>
 #include <unistd.h>
+#include <sys/resource.h>
 
 #if _POSIX_SPAWN
 #include <spawn.h>
@@ -95,6 +96,11 @@ int _was_process_signaled(int status);
 int _get_signal_code(int status);
 int _was_process_suspended(int status);
 
+/// Returns the soft RLIMIT_NOFILE value for the current process, or 0 on
+/// error.  Implemented in C so that RLIMIT_NOFILE always resolves to the
+/// correct type regardless of how the Swift Glibc/Darwin overlay imports it.
+uint64_t _subprocess_nofile_soft_limit(void);
+
 void _subprocess_lock_environ(void);
 void _subprocess_unlock_environ(void);
 char * _Nullable * _Nullable _subprocess_get_environ(void);
diff --git a/Sources/_SubprocessCShims/process_shims.c b/Sources/_SubprocessCShims/process_shims.c
@@ -78,6 +78,14 @@ int _was_process_suspended(int status) {
     return WIFSTOPPED(status);
 }
 
+uint64_t _subprocess_nofile_soft_limit(void) {
+    struct rlimit rl;
+    if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
+        return 0;
+    }
+    return (uint64_t)rl.rlim_cur;
+}
+
 int _subprocess_pthread_create(
     pthread_t * _Nonnull ptr,
     pthread_attr_t const * _Nullable attr,
@@ -584,6 +592,7 @@ int _subprocess_fork_exec(
     if (rc != 0) {
         close(pipefd[0]);
         close(pipefd[1]);
+        pthread_mutex_unlock(&_subprocess_fork_lock);
         return errno;
     }
 
@@ -602,6 +611,7 @@ int _subprocess_fork_exec(
             // Report all other errors
             close(pipefd[0]);
             close(pipefd[1]);
+            pthread_mutex_unlock(&_subprocess_fork_lock);
             return errno;
         }
     }
@@ -610,6 +620,7 @@ int _subprocess_fork_exec(
         // Fork failed
         close(pipefd[0]);
         close(pipefd[1]);
+        pthread_mutex_unlock(&_subprocess_fork_lock);
         return errno;
     }
 
@@ -765,6 +776,7 @@ int _subprocess_fork_exec(
         // Restore old signmask
         rc = pthread_sigmask(SIG_SETMASK, &old_sigmask, NULL);
         if (rc != 0) {
+            pthread_mutex_unlock(&_subprocess_fork_lock);
             reap_child_process_and_return_errno;
         }
 
diff --git a/Tests/SubprocessTests/UnixTests.swift b/Tests/SubprocessTests/UnixTests.swift
@@ -929,52 +929,60 @@ extension FileDescriptor {
 extension SubprocessUnixTests {
     #if SubprocessFoundation
     @Test(.requiresBash) func testConcurrentRun() async throws {
-        // Launch as many processes as we can
-        // Figure out the max open file limit
-        let limitResult = try await Subprocess.run(
-            .path("/bin/sh"),
-            arguments: ["-c", "ulimit -n"],
-            output: .string(limit: 32)
-        )
-        guard
-            let limitString = limitResult
-                .standardOutput?
-                .trimmingCharacters(in: .whitespacesAndNewlines),
-            let ulimit = Int(limitString)
-        else {
-            Issue.record("Failed to run  ulimit -n")
-            return
-        }
-        // Constrain to an ultimate upper limit of 4096, since Docker containers can have limits like 2^20 which is a bit too high for this test.
-        // Common defaults are 2560 for macOS and 1024 for Linux.
-        let limit = min(ulimit, 4096)
-        // Since we open two pipes per `run`, launch
-        // limit / 4 subprocesses should reveal any
-        // file descriptor leaks
-        let maxConcurrent = limit / 4
+        // Read the soft fd limit via a C shim: RLIMIT_NOFILE's Swift type
+        // varies across platforms and Swift versions, so calling getrlimit
+        // directly from Swift is not reliably portable.
+        // Cap at 4096: Docker containers can report limits like 2^20.
+        let softLimit = Int(min(_subprocess_nofile_soft_limit(), UInt64(4096)))
+
+        // On Linux, account for any fds already open (e.g. from prior tests in
+        // the same suite) to avoid hitting EMFILE during the concurrent spawn
+        // burst.  /proc/self/fd lists every open descriptor; subtracting the
+        // current count plus a small margin gives the true available headroom.
+        #if os(Linux) || os(Android)
+        let currentFds = (try? FileManager.default.contentsOfDirectory(atPath: "/proc/self/fd"))?.count ?? 50
+        let available = max(32, softLimit - currentFds - 50)
+        #else
+        let available = softLimit
+        #endif
+        // Each concurrent spawn holds both ends of the stdout and stderr pipes
+        // plus a temporary exec-error notification pipe while the child's
+        // exec() completes — roughly 6–8 fds per in-flight spawn.  Divide by
+        // 8 to leave headroom and avoid EMFILE under high concurrency.
+        let maxConcurrent = available / 8
         try await withThrowingTaskGroup(of: Void.self) { group in
             var running = 0
             let byteCount = 1000
             for _ in 0..<maxConcurrent {
                 group.addTask {
-                    // This invocation specifically requires bash semantics; sh (on FreeBSD at least) does not consistently support -s in this way
-                    let r = try await Subprocess.run(
-                        .name("bash"),
-                        arguments: [
-                            "-sc", #"echo "$1" && echo "$1" >&2"#, "--", String(repeating: "X", count: byteCount),
-                        ],
-                        output: .data(limit: .max),
-                        error: .data(limit: .max)
-                    )
-                    guard r.terminationStatus.isSuccess else {
-                        Issue.record("Unexpected exit \(r.terminationStatus) from \(r.processIdentifier)")
-                        return
+                    // Catch errors so a single spawn/monitor failure doesn't
+                    // cascade-cancel sibling tasks (which would SIGKILL their
+                    // live subprocesses and flood the log with false failures).
+                    do {
+                        // This invocation specifically requires bash semantics; sh (on FreeBSD at least) does not consistently support -s in this way
+                        let r = try await Subprocess.run(
+                            .name("bash"),
+                            arguments: [
+                                "-sc", #"echo "$1" && echo "$1" >&2"#, "--", String(repeating: "X", count: byteCount),
+                            ],
+                            output: .data(limit: .max),
+                            error: .data(limit: .max)
+                        )
+                        guard r.terminationStatus.isSuccess else {
+                            Issue.record("Unexpected exit \(r.terminationStatus) from \(r.processIdentifier)")
+                            return
+                        }
+                        #expect(r.standardOutput.count == byteCount + 1, "\(r.standardOutput)")
+                        #expect(r.standardError.count == byteCount + 1, "\(r.standardError)")
+                    } catch {
+                        Issue.record("Subprocess.run threw: \(error)")
                     }
-                    #expect(r.standardOutput.count == byteCount + 1, "\(r.standardOutput)")
-                    #expect(r.standardError.count == byteCount + 1, "\(r.standardError)")
                 }
                 running += 1
-                if running >= maxConcurrent / 4 {
+                // Throttle to maxConcurrent/8 live subprocesses at a time
+                // (rather than /4) to reduce peak memory pressure on
+                // memory-constrained kernel-testing VMs (e.g. QEMU + 5.10).
+                if running >= maxConcurrent / 8 {
                     try await group.next()
                 }
             }
diff --git a/scripts/prep-linux-swift.sh b/scripts/prep-linux-swift.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the Swift.org open source project
+##
+## Copyright (c) 2026 Apple Inc. and the Swift project authors
+## Licensed under Apache License v2.0 with Runtime Library Exception
+##
+## See https://swift.org/LICENSE.txt for license information
+##
+##===----------------------------------------------------------------------===##
+
+# This script does a bit of extra preparation of the docker containers used to run the GitHub workflows
+# that are specific to this project's needs when building/testing. Note that this script runs on
+# every supported Linux distribution so it must adapt to the distribution that it is running.
+
+if [[ "$(uname -s)" == "Linux" ]]; then
+    # Install the basic utilities depending on the type of Linux distribution
+    apt-get --help && apt-get update && TZ=Etc/UTC apt-get -y install curl make gpg tzdata
+    yum --help && (curl --help && yum -y install curl) && yum -y install make gpg tar procps
+fi
+
+set -e
+
+while [ $# -ne 0 ]; do
+    arg="$1"
+    case "$arg" in
+        --install-swiftly)
+            installSwiftly=true
+            ;;
+        --swift-snapshot)
+            swiftSnapshot="$2"
+            shift;
+            ;;
+        *)
+            ;;
+    esac
+    shift
+done
+
+if [ "$installSwiftly" == true ]; then
+    echo "Installing swiftly"
+
+    curl -O "https://download.swift.org/swiftly/linux/swiftly-$(uname -m).tar.gz" && tar zxf swiftly-*.tar.gz && ./swiftly init -y --skip-install
+    # shellcheck source=/dev/null
+    . "/root/.local/share/swiftly/env.sh"
+
+    hash -r
+
+    selector=()
+    runSelector=()
+
+    if [ "$swiftSnapshot" != "" ]; then
+        echo "Installing latest $swiftSnapshot-snapshot toolchain"
+        selector=("$swiftSnapshot-snapshot")
+        runSelector=("+$swiftSnapshot-snapshot")
+    elif [ -f .swift-version ]; then
+        echo "Installing selected swift toolchain from .swift-version file"
+        selector=()
+        runSelector=()
+    else
+        echo "Installing latest toolchain"
+        selector=("latest")
+        runSelector=("+latest")
+    fi
+
+    TMPDIR=/var/tmp swiftly install --post-install-file=post-install.sh "${selector[@]}"
+
+    if [ -f post-install.sh ]; then
+        echo "Performing swift toolchain post-installation"
+        chmod u+x post-install.sh && ./post-install.sh
+    fi
+
+    echo "Displaying swift version"
+    swiftly run "${runSelector[@]}" swift --version
+fi
diff --git a/scripts/test-using-qemu.sh b/scripts/test-using-qemu.sh
