Skip to content

Commit 09f8954

Browse files
committed
Resolve container native library lookup
- Prefer target process root for live-process absolute module paths before host path lookup. - Keep build-id lookup first and preserve libdwfl handling for deleted and non-absolute mappings. - Add a namespace integration test and require it in the Ubuntu coverage job. Signed-off-by: Pramod Kumbhar <prkumbhar@nvidia.com>
1 parent 4d4ef00 commit 09f8954

3 files changed

Lines changed: 335 additions & 3 deletions

File tree

.github/workflows/coverage.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ jobs:
4040
sudo apt-get install -qy \
4141
gdb \
4242
lcov \
43+
gcc \
4344
cmake \
4445
ninja-build \
4546
libdw-dev \
@@ -53,6 +54,14 @@ jobs:
5354
- name: Disable ptrace security restrictions
5455
run: |
5556
echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
57+
- name: Require process-root native-symbol test support
58+
run: |
59+
sudo sysctl -w kernel.unprivileged_userns_clone=1 || true
60+
sudo sysctl -w user.max_user_namespaces=15000 || true
61+
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true
62+
unshare --user --map-root-user --mount --propagation private true
63+
PYSTACK_REQUIRE_PROCESS_ROOT_TEST=1 \
64+
python3 -m pytest -vvv -s tests/integration/test_native_process_root.py
5665
- name: Compute Python coverage
5766
run: |
5867
python3 -m pytest -vvv --log-cli-level=info -s --color=yes \

src/pystack/_pystack/elf_common.cpp

Lines changed: 73 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <inttypes.h>
55
#include <iomanip>
66
#include <iostream>
7+
#include <sys/stat.h>
78
#include <string>
89
#include <utility>
910

@@ -14,6 +15,64 @@ namespace pystack {
1415

1516
using file_unique_ptr = std::unique_ptr<FILE, std::function<int(FILE*)>>;
1617

18+
static int
19+
set_module_process_pid_userdata(
20+
Dwfl_Module* mod __attribute__((unused)),
21+
void** userdata,
22+
const char* name __attribute__((unused)),
23+
Dwarf_Addr start __attribute__((unused)),
24+
void* arg)
25+
{
26+
*userdata = arg;
27+
return DWARF_CB_OK;
28+
}
29+
30+
static bool
31+
is_deleted_mapping(const char* modname)
32+
{
33+
const char* last_space = strrchr(modname, ' ');
34+
return last_space != nullptr && strcmp(last_space, " (deleted)") == 0;
35+
}
36+
37+
static bool
38+
should_find_elf_through_proc_pid_root(void** userdata, const char* modname)
39+
{
40+
return userdata != nullptr && *userdata != nullptr && modname != nullptr && modname[0] == '/' &&
41+
!is_deleted_mapping(modname);
42+
}
43+
44+
// Open a mapped path relative to the target process root.
45+
static int
46+
find_elf_through_proc_pid_root(void** userdata, const char* modname, char** file_name)
47+
{
48+
if (!should_find_elf_through_proc_pid_root(userdata, modname)) {
49+
return -1;
50+
}
51+
52+
const auto pid = *static_cast<const int*>(*userdata);
53+
const std::string rooted_path = "/proc/" + std::to_string(pid) + "/root" + modname;
54+
struct stat sb;
55+
if (stat(rooted_path.c_str(), &sb) == -1 || (sb.st_mode & S_IFMT) != S_IFREG) {
56+
return -1;
57+
}
58+
59+
int fd = open(rooted_path.c_str(), O_RDONLY);
60+
if (fd < 0) {
61+
return -1;
62+
}
63+
if (file_name == nullptr) {
64+
return fd;
65+
}
66+
67+
*file_name = strdup(rooted_path.c_str());
68+
if (*file_name == nullptr) {
69+
close(fd);
70+
return -1;
71+
}
72+
73+
return fd;
74+
}
75+
1776
int
1877
pystack_find_elf(
1978
Dwfl_Module* mod,
@@ -30,11 +89,17 @@ pystack_find_elf(
3089
LOG(DEBUG) << "Located debug info for " << the_modname << " using BUILD ID in " << the_filename;
3190
return ret;
3291
}
33-
ret = dwfl_linux_proc_find_elf(mod, userdata, modname, base, file_name, elfp);
34-
if (file_name == nullptr) {
92+
93+
const bool use_process_root = should_find_elf_through_proc_pid_root(userdata, modname);
94+
ret = use_process_root ? find_elf_through_proc_pid_root(userdata, modname, file_name) : -1;
95+
if (ret < 0 && !use_process_root) {
96+
ret = dwfl_linux_proc_find_elf(mod, userdata, modname, base, file_name, elfp);
97+
}
98+
if (ret < 0) {
3599
LOG(DEBUG) << "Could not locate debug info for " << the_modname;
36100
} else {
37-
LOG(DEBUG) << "Located debug info for " << the_modname << " by path in " << *file_name;
101+
const char* the_filename = (file_name == nullptr || *file_name == nullptr) ? "???" : *file_name;
102+
LOG(DEBUG) << "Located debug info for " << the_modname << " by path in " << the_filename;
38103
}
39104
return ret;
40105
}
@@ -285,6 +350,11 @@ ProcessAnalyzer::ProcessAnalyzer(pid_t pid)
285350
throw ElfAnalyzerError("Failed to analyze DWARF information for the remote process");
286351
}
287352

353+
// The find_elf callback needs the PID to retry module paths through /proc/<pid>/root.
354+
if (dwfl_getmodules(d_dwfl.get(), set_module_process_pid_userdata, &d_pid, 0) == -1) {
355+
throw ElfAnalyzerError("Failed to associate DWARF modules with the remote process");
356+
}
357+
288358
if (dwfl_linux_proc_attach(d_dwfl.get(), pid, true) != 0) {
289359
throw ElfAnalyzerError("Could not attach the DWARF process analyzer");
290360
}
Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
import os
2+
import shlex
3+
import shutil
4+
import subprocess
5+
import sys
6+
import textwrap
7+
import time
8+
from pathlib import Path
9+
10+
import pytest
11+
12+
from pystack.engine import NativeReportingMode
13+
from pystack.engine import get_process_threads
14+
15+
16+
LIBRARY_NAME = "libpystack_process_root_test.so"
17+
REQUIRE_PROCESS_ROOT_TEST = os.environ.get("PYSTACK_REQUIRE_PROCESS_ROOT_TEST") == "1"
18+
19+
20+
def skip_or_fail(reason: str) -> None:
21+
if REQUIRE_PROCESS_ROOT_TEST:
22+
pytest.fail(reason)
23+
pytest.skip(reason)
24+
25+
26+
def compiler_command() -> list[str] | None:
27+
compiler = os.environ.get("CC")
28+
if compiler:
29+
return shlex.split(compiler)
30+
31+
compiler = shutil.which("cc") or shutil.which("gcc")
32+
if compiler is None:
33+
return None
34+
return [compiler]
35+
36+
37+
def compile_shared_library(
38+
compiler: list[str],
39+
source: Path,
40+
output: Path,
41+
symbol: str,
42+
) -> None:
43+
source.write_text(
44+
textwrap.dedent(
45+
f"""
46+
#include <fcntl.h>
47+
#include <unistd.h>
48+
49+
__attribute__((noinline)) void
50+
{symbol}(const char* ready_path)
51+
{{
52+
int fd = open(ready_path, O_WRONLY | O_CREAT | O_TRUNC, 0666);
53+
if (fd >= 0) {{
54+
write(fd, "ready", 5);
55+
close(fd);
56+
}}
57+
sleep(1000);
58+
}}
59+
"""
60+
)
61+
)
62+
subprocess.run(
63+
[
64+
*compiler,
65+
"-g",
66+
"-O0",
67+
"-fno-omit-frame-pointer",
68+
"-fPIC",
69+
"-shared",
70+
"-o",
71+
str(output),
72+
str(source),
73+
],
74+
check=True,
75+
)
76+
77+
78+
def unshare_mount_namespace_command() -> list[str] | None:
79+
unshare = shutil.which("unshare")
80+
if unshare is None:
81+
return None
82+
83+
candidates = []
84+
if os.geteuid() == 0:
85+
candidates.append([unshare, "--mount", "--propagation", "private"])
86+
candidates.append(
87+
[
88+
unshare,
89+
"--user",
90+
"--map-root-user",
91+
"--mount",
92+
"--propagation",
93+
"private",
94+
]
95+
)
96+
97+
for command in candidates:
98+
if (
99+
subprocess.run(
100+
[
101+
*command,
102+
"true",
103+
],
104+
stdout=subprocess.DEVNULL,
105+
stderr=subprocess.DEVNULL,
106+
).returncode
107+
== 0
108+
):
109+
return command
110+
111+
return None
112+
113+
114+
def wait_until_ready(process: subprocess.Popen, ready_file: Path) -> None:
115+
deadline = time.monotonic() + 10
116+
while time.monotonic() < deadline:
117+
if ready_file.exists():
118+
return
119+
if process.poll() is not None:
120+
_, stderr = process.communicate()
121+
message = stderr.strip()
122+
if "Operation not permitted" in message or "permission denied" in message.lower():
123+
skip_or_fail(f"mount namespace setup is not permitted: {message}")
124+
pytest.fail(f"target process exited before it was ready: {message}")
125+
time.sleep(0.1)
126+
127+
process.terminate()
128+
pytest.fail("timed out waiting for target process")
129+
130+
131+
@pytest.mark.skipif(sys.platform != "linux", reason="requires Linux procfs")
132+
def test_native_symbols_use_target_process_root(tmp_path: Path) -> None:
133+
compiler = compiler_command()
134+
if compiler is None:
135+
skip_or_fail("a C compiler is required to build the test shared libraries")
136+
if shutil.which("mount") is None:
137+
skip_or_fail("mount is required to set up the private mount namespace")
138+
139+
unshare_command = unshare_mount_namespace_command()
140+
if unshare_command is None:
141+
skip_or_fail("user and mount namespaces are not available")
142+
143+
target_dir = tmp_path / "target"
144+
host_dir = tmp_path / "mapped"
145+
target_dir.mkdir()
146+
host_dir.mkdir()
147+
148+
target_library = target_dir / LIBRARY_NAME
149+
host_library = host_dir / LIBRARY_NAME
150+
mapped_library = host_library
151+
152+
compile_shared_library(
153+
compiler,
154+
tmp_path / "target.c",
155+
target_library,
156+
"pystack_target_process_root_symbol",
157+
)
158+
compile_shared_library(
159+
compiler,
160+
tmp_path / "host.c",
161+
host_library,
162+
"pystack_host_decoy_symbol",
163+
)
164+
165+
target_program = tmp_path / "target_program.py"
166+
target_program.write_text(
167+
textwrap.dedent(
168+
"""
169+
import ctypes
170+
import os
171+
import sys
172+
from pathlib import Path
173+
174+
library = ctypes.CDLL(sys.argv[2])
175+
symbol = library.pystack_target_process_root_symbol
176+
symbol.argtypes = (ctypes.c_char_p,)
177+
symbol.restype = None
178+
symbol(os.fsencode(Path(sys.argv[1])))
179+
"""
180+
)
181+
)
182+
183+
namespace_launcher = tmp_path / "namespace_launcher.py"
184+
namespace_launcher.write_text(
185+
textwrap.dedent(
186+
"""
187+
import os
188+
import subprocess
189+
import sys
190+
from pathlib import Path
191+
192+
target_dir = sys.argv[1]
193+
host_dir = sys.argv[2]
194+
ready_file = sys.argv[3]
195+
target_program = sys.argv[4]
196+
library_name = sys.argv[5]
197+
198+
subprocess.run(["mount", "--bind", target_dir, host_dir], check=True)
199+
mapped_library = str(Path(host_dir) / library_name)
200+
os.execv(
201+
sys.executable,
202+
[sys.executable, "-S", target_program, ready_file, mapped_library],
203+
)
204+
"""
205+
)
206+
)
207+
208+
ready_file = tmp_path / "ready"
209+
process = subprocess.Popen(
210+
[
211+
*unshare_command,
212+
sys.executable,
213+
"-S",
214+
str(namespace_launcher),
215+
str(target_dir),
216+
str(host_dir),
217+
str(ready_file),
218+
str(target_program),
219+
LIBRARY_NAME,
220+
],
221+
stdout=subprocess.PIPE,
222+
stderr=subprocess.PIPE,
223+
text=True,
224+
)
225+
226+
try:
227+
wait_until_ready(process, ready_file)
228+
process_root_library = Path(f"/proc/{process.pid}/root") / str(
229+
mapped_library
230+
).lstrip(os.sep)
231+
232+
assert mapped_library.exists()
233+
assert process_root_library.exists()
234+
assert os.stat(mapped_library).st_ino != os.stat(process_root_library).st_ino
235+
236+
threads = list(
237+
get_process_threads(
238+
process.pid,
239+
native_mode=NativeReportingMode.PYTHON,
240+
stop_process=True,
241+
)
242+
)
243+
244+
symbols = {frame.symbol for thread in threads for frame in thread.native_frames}
245+
assert "pystack_target_process_root_symbol" in symbols
246+
assert "pystack_host_decoy_symbol" not in symbols
247+
finally:
248+
process.terminate()
249+
try:
250+
process.wait(timeout=5)
251+
except subprocess.TimeoutExpired:
252+
process.kill()
253+
process.wait(timeout=5)

0 commit comments

Comments
 (0)