Skip to content

Commit 6534cf1

Browse files
danbugsdanbugs
authored
feat: subprocess support, networking idle-poll fix, and CI improvements (#108)
* feat(kernel): point examples at danbugs/unikraft fork Signed-off-by: danbugs <danilochiarlone@gmail.com> * feat(host): increase heap to 2.5 GiB for subprocess support Signed-off-by: danbugs <danilochiarlone@gmail.com> * fix(pydriver): replace strtoul with inline parse_hex Signed-off-by: danbugs <danilochiarlone@gmail.com> * feat(python-agent-driver): add subprocess demos and rootfs Signed-off-by: danbugs <danilochiarlone@gmail.com> * ci: add subprocess demo tests to regression gate Signed-off-by: danbugs <danilochiarlone@gmail.com> * fix(python-agent-driver): build base images automatically in just rootfs Signed-off-by: danbugs <danilochiarlone@gmail.com> * test: add urllib GET without timeout CI test Signed-off-by: danbugs <danilochiarlone@gmail.com> * ci: add per-test timeout to Windows runtime tests Signed-off-by: danbugs <danilochiarlone@gmail.com> * fix(kernel): point kraft.yaml at upstream plat-hyperlight Signed-off-by: danbugs <danilochiarlone@gmail.com> * fix(python-agent-driver): remove pre-bundled wheel, use toml for pip demo Signed-off-by: danbugs <danilochiarlone@gmail.com> * fix(python-agent-driver): remove stale COPY of deleted /wheels dir Signed-off-by: danbugs <danilochiarlone@gmail.com> * fix(kernel): revert kraft.yaml to fork branches until upstream PRs merge Signed-off-by: danbugs <danilochiarlone@gmail.com> * fix(kernel): point all examples at upstream plat-hyperlight Signed-off-by: danbugs <danilochiarlone@gmail.com> * ci: retrigger after upstream signal guard fix Signed-off-by: danbugs <danilochiarlone@gmail.com> * ci: retrigger CI Signed-off-by: danbugs <danilochiarlone@gmail.com> * fix(kernel): point kraft.yaml at epoll bypass fix branch Signed-off-by: danbugs <danilochiarlone@gmail.com> * fix(ci): capture stderr in Windows Invoke-WithTimeout hyperlight-unikraft writes guest output to stderr with -q flag. The Linux test merges stderr via 2>&1, but the Windows helper only read stdout. Merge both streams so output matching works on Windows. Signed-off-by: danbugs <danilochiarlone@gmail.com> * fix(kernel): point kraft.yaml at upstream plat-hyperlight PR unikraft/unikraft#1860 (epoll bypass guard) is now merged. Signed-off-by: danbugs <danilochiarlone@gmail.com> --------- Signed-off-by: danbugs <danilochiarlone@gmail.com>
1 parent f1c1281 commit 6534cf1

11 files changed

Lines changed: 183 additions & 26 deletions

File tree

‎.github/workflows/test-examples.yml‎

Lines changed: 58 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,9 @@ jobs:
223223
- example: networking-py
224224
args: "--net -- /urllib_get.py"
225225
expect: "SUCCESS: urllib GET worked!"
226+
- example: networking-py
227+
args: "--net -- /urllib_get_no_timeout.py"
228+
expect: "SUCCESS: urllib GET \\(no timeout\\) worked!"
226229
- example: networking-py
227230
args: "--port 8080 -- /echo_server_test.py"
228231
expect: "SUCCESS: bind\\+listen on port 8080 allowed"
@@ -613,6 +616,9 @@ jobs:
613616
- example: networking-py
614617
args: "--net -- /urllib_get.py"
615618
expect: "SUCCESS: urllib GET worked!"
619+
- example: networking-py
620+
args: "--net -- /urllib_get_no_timeout.py"
621+
expect: "SUCCESS: urllib GET \\(no timeout\\) worked!"
616622
- example: networking-py
617623
args: "--port 8080 -- /echo_server_test.py"
618624
expect: "SUCCESS: bind\\+listen on port 8080 allowed"
@@ -753,16 +759,33 @@ jobs:
753759
}
754760
}
755761
762+
# Helper: run a command with a per-test timeout (seconds).
763+
function Invoke-WithTimeout {
764+
param([int]$Seconds, [string]$Exe, [string[]]$ArgList)
765+
$outFile = Join-Path $env:RUNNER_TEMP 'hl-test-out.log'
766+
$proc = Start-Process -FilePath $Exe -ArgumentList $ArgList `
767+
-NoNewWindow -RedirectStandardOutput $outFile -RedirectStandardError "$outFile.err" -PassThru
768+
if (-not $proc.WaitForExit($Seconds * 1000)) {
769+
Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue
770+
Write-Host "FAIL: timed out after ${Seconds}s"
771+
Get-Content $outFile -ErrorAction SilentlyContinue
772+
Get-Content "$outFile.err" -ErrorAction SilentlyContinue
773+
exit 1
774+
}
775+
$stdoutContent = Get-Content $outFile -Raw -ErrorAction SilentlyContinue
776+
$stderrContent = Get-Content "$outFile.err" -Raw -ErrorAction SilentlyContinue
777+
$script:out = "$stdoutContent$stderrContent".Trim()
778+
$script:rc = $proc.ExitCode
779+
}
780+
756781
switch ($driver) {
757782
'multifn-test' {
758-
$out = & multifn-test $kernel $cpio 2>&1
759-
$rc = $LASTEXITCODE
783+
Invoke-WithTimeout -Seconds 60 -Exe 'multifn-test' -ArgList @($kernel, $cpio)
760784
}
761785
'pydriver-run' {
762786
"print('hello from driver')" | Out-File -Encoding ascii tiny.py
763787
$tiny = (Resolve-Path "tiny.py").Path
764-
$out = & pydriver-run $kernel $cpio $tiny 2>&1
765-
$rc = $LASTEXITCODE
788+
Invoke-WithTimeout -Seconds 120 -Exe 'pydriver-run' -ArgList @($kernel, $cpio, $tiny)
766789
}
767790
default {
768791
$argList = @()
@@ -773,16 +796,15 @@ jobs:
773796
if ($memory -ne '') {
774797
$memArgs = @('-m', $memory)
775798
}
776-
$out = & hyperlight-unikraft -q @memArgs `
777-
$kernel --initrd $cpio @mountArgs @toolArgs @argList 2>&1
778-
$rc = $LASTEXITCODE
799+
Invoke-WithTimeout -Seconds 120 -Exe 'hyperlight-unikraft' `
800+
-ArgList (@('-q') + $memArgs + @($kernel, '--initrd', $cpio) + $mountArgs + $toolArgs + $argList)
779801
}
780802
}
781803
782-
$outStr = ($out | Out-String)
804+
if ($null -eq $out) { $out = '' }
783805
Write-Host "=== output (exit=$rc) ==="
784-
Write-Host $outStr
785-
if (-not ($outStr -match $expect)) {
806+
Write-Host $out
807+
if (-not ($out -match $expect)) {
786808
Write-Host "FAIL: did not match /$expect/"
787809
exit 1
788810
}
@@ -946,6 +968,32 @@ jobs:
946968
exit 1
947969
}
948970
971+
- name: pyhl run (busybox subprocess demo)
972+
if: runner.os == 'Windows' || steps.kvm_check.outputs.available == 'true'
973+
shell: pwsh
974+
run: |
975+
$out = pyhl run examples/python-agent-driver/demo_busybox.py 2>&1
976+
Write-Host $out
977+
if ($out -match 'hello from hyperlight guest') {
978+
Write-Host "PASS: busybox subprocess demo"
979+
} else {
980+
Write-Error "FAIL: did not match /hello from hyperlight guest/"
981+
exit 1
982+
}
983+
984+
- name: pyhl run (pip install subprocess demo)
985+
if: runner.os == 'Windows' || steps.kvm_check.outputs.available == 'true'
986+
shell: pwsh
987+
run: |
988+
$out = pyhl run --net examples/python-agent-driver/demo_pip_install.py 2>&1
989+
Write-Host $out
990+
if ($out -match 'Installed and imported six') {
991+
Write-Host "PASS: pip install subprocess demo"
992+
} else {
993+
Write-Error "FAIL: did not match /Installed and imported six/"
994+
exit 1
995+
}
996+
949997
test-examples-passed:
950998
if: always()
951999
needs:

‎examples/networking-py/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ FROM ${BASE} AS rootfs
88
COPY http_get.py /http_get.py
99
COPY echo_server.py /echo_server.py
1010
COPY urllib_get.py /urllib_get.py
11+
COPY urllib_get_no_timeout.py /urllib_get_no_timeout.py
1112
COPY https_test.py /https_test.py
1213
COPY echo_server_test.py /echo_server_test.py
1314

‎examples/networking-py/urllib_get_no_timeout.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
"""HTTP GET using urllib WITHOUT an explicit timeout.
2+
3+
Same as urllib_get.py but omits the timeout= argument to urlopen().
4+
This exercises the code path used by mxc, where the Unikraft guest
5+
kernel relies on the idle thread's halt_irq callback to poll sockets
6+
via __hl_sleep rather than an explicit timeout-driven poll cycle.
7+
"""
8+
import urllib.request
9+
import sys
10+
11+
URL = "http://example.com/"
12+
13+
print(f"Fetching {URL} (no timeout) ...")
14+
try:
15+
with urllib.request.urlopen(URL) as resp:
16+
body = resp.read().decode("utf-8", errors="replace")
17+
print(f"Status: {resp.status}")
18+
print(f"Body length: {len(body)} bytes")
19+
if "Example Domain" in body:
20+
print("SUCCESS: urllib GET (no timeout) worked!")
21+
else:
22+
print("WARNING: unexpected body content")
23+
except Exception as e:
24+
print(f"FAILED: {e}")
25+
sys.exit(1)

‎examples/python-agent-driver/Dockerfile‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,17 @@ COPY --from=deps /deps /usr/local/lib/python3.12/site-package
7171
COPY --from=driver-build /src/hl_pydriver /bin/hl_pydriver
7272
COPY pydoc_stub.py /usr/local/lib/python3.12/pydoc.py
7373

74+
# pip: the python-base image strips pip for size; bring it back from the
75+
# deps stage (python:3.12-slim) so `python3 -m pip install` works as a
76+
# subprocess.
77+
COPY --from=deps /usr/local/lib/python3.12/site-packages/pip \
78+
/usr/local/lib/python3.12/site-packages/pip
79+
80+
# stdlib modules that python-base strips but pip needs at runtime
81+
COPY --from=deps /usr/local/lib/python3.12/xmlrpc \
82+
/usr/local/lib/python3.12/xmlrpc
83+
84+
7485
# Stage 4: pack CPIO.
7586
FROM alpine:3.20 AS cpio
7687
RUN apk add --no-cache cpio findutils ca-certificates
@@ -88,4 +99,22 @@ RUN mkdir -p /rootfs/etc/ssl/certs /rootfs/usr/lib/ssl && \
8899
cp /etc/ssl/certs/ca-certificates.crt /rootfs/etc/ssl/certs/ && \
89100
ln -sf /etc/ssl/certs/ca-certificates.crt /rootfs/usr/lib/ssl/cert.pem
90101

102+
# pip configuration: disable version-check (no internet during warm-up)
103+
RUN mkdir -p /rootfs/etc && \
104+
printf '[global]\ndisable-pip-version-check = true\n' > /rootfs/etc/pip.conf
105+
106+
# /tmp for pip's temp files during install
107+
RUN mkdir -p /rootfs/tmp
108+
109+
# Busybox: use the static-PIE build from the shell-base image.
110+
# Alpine's busybox-static is ET_EXEC (non-PIE) which the elfloader
111+
# rejects; the shell-base image has a static-PIE (ET_DYN) build.
112+
COPY --from=ghcr.io/hyperlight-dev/hyperlight-unikraft/shell-base:latest \
113+
/bin/busybox /rootfs/bin/busybox
114+
RUN for cmd in sh echo ls cat grep find wc head tail sort cut sed awk \
115+
cp mv rm mkdir rmdir ln basename dirname env sleep date \
116+
hostname id uname whoami which test true tr uniq tee xargs; do \
117+
ln -sf busybox /rootfs/bin/$cmd; \
118+
done
119+
91120
RUN cd /rootfs && find . | cpio -o -H newc > /output.cpio 2>/dev/null

‎examples/python-agent-driver/Justfile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ run-5:
3030

3131
[unix]
3232
rootfs:
33+
docker build --target base -t local-python-base-dev:latest \
34+
-f ../../runtimes/python.Dockerfile ../../runtimes/
35+
docker build -t local-python-base:latest \
36+
-f ../../runtimes/python.Dockerfile ../../runtimes/
3337
docker build --platform linux/amd64 --build-arg BASE=local-python-base:latest \
3438
--target cpio -t {{image}}-cpio .
3539
- docker rm -f {{image}}-tmp

‎examples/python-agent-driver/demo_busybox.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import subprocess
2+
3+
cmds = [
4+
(["echo", "hello from hyperlight guest"], None),
5+
(["uname", "-a"], None),
6+
(["ls", "/bin"], None),
7+
(["grep", "nameserver", "/etc/resolv.conf"], None),
8+
(["find", "/etc", "-name", "*.conf"], None),
9+
(["wc", "-l", "/etc/resolv.conf"], None),
10+
(["sh", "-c", "echo hello from sh"], None),
11+
]
12+
13+
for cmd, stdin in cmds:
14+
label = " ".join(cmd)
15+
print(f"\n$ {label}")
16+
r = subprocess.run(cmd, capture_output=True, text=True, input=stdin)
17+
if r.stdout:
18+
print(r.stdout.rstrip())
19+
if r.stderr:
20+
print(f"stderr: {r.stderr.rstrip()}")
21+
if r.returncode != 0:
22+
print(f"exit code: {r.returncode}")

‎examples/python-agent-driver/demo_pip_install.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import subprocess, sys, os
2+
3+
target = "/tmp/pypackages"
4+
os.makedirs(target, exist_ok=True)
5+
6+
result = subprocess.run(
7+
[sys.executable, "-m", "pip", "install", "--target", target, "six"],
8+
capture_output=True, text=True,
9+
)
10+
print(result.stdout)
11+
if result.returncode != 0:
12+
print(result.stderr)
13+
sys.exit(result.returncode)
14+
15+
sys.path.insert(0, target)
16+
import six
17+
print(f"Installed and imported six {six.__version__}")

‎examples/python-agent-driver/hl_pydriver.c‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -110,18 +110,28 @@ static const char *fc_arg0_string(const uint8_t *fc, size_t fc_len,
110110
* FC-aware callback the kernel dispatches to
111111
* on every call after it's set
112112
*/
113+
static uintptr_t parse_hex(const char *s)
114+
{
115+
uintptr_t v = 0;
116+
if (s[0] == '0' && (s[1] == 'x' || s[1] == 'X'))
117+
s += 2;
118+
for (; *s; s++) {
119+
unsigned d;
120+
if (*s >= '0' && *s <= '9') d = *s - '0';
121+
else if (*s >= 'a' && *s <= 'f') d = *s - 'a' + 10;
122+
else if (*s >= 'A' && *s <= 'F') d = *s - 'A' + 10;
123+
else break;
124+
v = (v << 4) | d;
125+
}
126+
return v;
127+
}
128+
113129
typedef void (*hl_dispatch_fn_t)(const uint8_t *fc, size_t fc_len);
114130

115131
static const uint8_t **g_fc_bytes_slot;
116132
static size_t *g_fc_len_slot;
117133
static hl_dispatch_fn_t *g_v2_callback_slot;
118134

119-
/* Saved FS_BASE value captured right after Py_Initialize / warm-up
120-
* finishes. Restored at the head of every v2-callback invocation so
121-
* Python's TLS pointer stays valid even if something in the dispatch
122-
* preamble (dispatch_prepare's MSR restore, Hyperlight's own
123-
* save/restore of segment state) leaves FS_BASE pointing elsewhere.
124-
*/
125135
static uint64_t g_py_fsbase;
126136

127137
static inline uint64_t rdmsr_fsbase(void)
@@ -158,7 +168,6 @@ static int run_code_with_exceptions(const char *code)
158168
if (!m) return 1;
159169
PyObject *d = PyModule_GetDict(m);
160170
if (!d) return 1;
161-
162171
PyObject *result = PyRun_String(code, Py_file_input, d, d);
163172
if (result) {
164173
Py_DECREF(result);
@@ -307,14 +316,14 @@ int main(int argc, char **argv, char **envp)
307316
if (!g_fc_bytes_slot) {
308317
for (char **p = envp; p && *p; p++) {
309318
if (!strncmp(*p, "HL_FC_BYTES_PTR=", 16))
310-
g_fc_bytes_slot = (const uint8_t **)(uintptr_t)
311-
strtoul(*p + 16, NULL, 16);
319+
g_fc_bytes_slot = (const uint8_t **)
320+
parse_hex(*p + 16);
312321
else if (!strncmp(*p, "HL_FC_LEN_PTR=", 14))
313-
g_fc_len_slot = (size_t *)(uintptr_t)
314-
strtoul(*p + 14, NULL, 16);
322+
g_fc_len_slot = (size_t *)
323+
parse_hex(*p + 14);
315324
else if (!strncmp(*p, "HL_V2_CALLBACK_PTR=", 19))
316-
g_v2_callback_slot = (hl_dispatch_fn_t *)(uintptr_t)
317-
strtoul(*p + 19, NULL, 16);
325+
g_v2_callback_slot = (hl_dispatch_fn_t *)
326+
parse_hex(*p + 19);
318327
}
319328
if (!g_fc_bytes_slot || !g_fc_len_slot
320329
|| !g_v2_callback_slot) {

‎examples/python-agent-driver/kraft.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,9 @@ unikraft:
7979
CONFIG_LIBCONTEXT: 'y'
8080
CONFIG_LIBCONTEXT_CLEAR_TBSS: 'y'
8181

82-
CONFIG_LIBPOSIX_FDTAB_MULTITAB: 'n'
82+
CONFIG_LIBPOSIX_FDTAB_MULTITAB: 'y'
83+
84+
CONFIG_LIBPOSIX_PROCESS_BRK: 'y'
8385

8486
libraries:
8587
app-elfloader:

‎host/src/bin/pydriver_run.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ fn main() -> Result<()> {
4444
let t_evolve = Instant::now();
4545
let mut sandbox = Sandbox::builder(&kernel)
4646
.initrd_file(&initrd)
47-
.heap_size(1280 * 1024 * 1024)
47+
.heap_size(2560 * 1024 * 1024)
4848
.build()?;
4949
eprintln!(
5050
"[timing] evolve={:.1}ms",

0 commit comments

Comments
 (0)