Skip to content

Commit 581b8d8

Browse files
danbugsdanbugs
authored
feat: expand python-agent-driver to 102 pip packages (#80)
* feat: expand python-agent-driver to 102 pip packages - Add 88 new pip packages to Dockerfile (removed edgartools due to pyarrow pulling in concurrent.futures.thread which crashes Unikraft) - Bump heap to 2.5 GiB across all entry points to accommodate the larger rootfs - Pre-import docx and pptx during warmup for zero-cost access - Add pydoc stub for pyarrow compatibility - Update --help to list all shipped packages - Add docs/python-packages.md reference Signed-off-by: danbugs <danilochiarlone@gmail.com> * ci: add smoke test and improve pandas benchmark - Add smoke test step before benchmarks to catch pyhl run failures early - Add explicit pandas smoke check before timing loop - Redirect stdout to /dev/null in timing runs to isolate timing output Signed-off-by: danbugs <danilochiarlone@gmail.com> --------- Signed-off-by: danbugs <danilochiarlone@gmail.com>
1 parent 7162172 commit 581b8d8

9 files changed

Lines changed: 231 additions & 25 deletions

File tree

‎.github/workflows/benchmarks.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,15 @@ jobs:
170170
if: steps.kvm_check.outputs.available == 'true'
171171
run: pyhl setup --from src-dir --force
172172

173+
- name: "Smoke: verify pyhl run works"
174+
if: steps.kvm_check.outputs.available == 'true'
175+
run: |
176+
echo "--- snapshot and system info ---"
177+
ls -lh .pyhl/
178+
free -h
179+
echo "--- smoke test ---"
180+
pyhl run -c "print('smoke ok')" || { echo "FAIL: pyhl run exited $?"; exit 1; }
181+
173182
- name: "Perf: hello world (15 runs)"
174183
if: steps.kvm_check.outputs.available == 'true'
175184
run: |
@@ -210,9 +219,10 @@ jobs:
210219
df = pd.DataFrame(np.random.randn(1000, 4), columns=list('ABCD'))
211220
print(df.describe())
212221
PYEOF
222+
pyhl run /tmp/pandas_bench.py > /dev/null || { echo "FAIL: pandas smoke check failed"; exit 1; }
213223
times=()
214224
for i in $(seq 1 10); do
215-
ms=$( { /usr/bin/time -f "%e" pyhl run /tmp/pandas_bench.py; } 2>&1 | tail -1 )
225+
ms=$( { /usr/bin/time -f "%e" pyhl run /tmp/pandas_bench.py > /dev/null; } 2>&1 )
216226
ms_int=$(echo "$ms * 1000" | bc | cut -d. -f1)
217227
times+=($ms_int)
218228
echo " run $i: ${ms_int}ms"

‎docs/python-packages.md‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
# Python package support
2+
3+
The python-agent-driver ships CPython 3.12 with the full standard library and 102 explicitly installed top-level pip packages (transitive dependencies are also included).
4+
5+
## Standard library
6+
7+
The complete CPython 3.12 standard library is included. Commonly used modules:
8+
9+
| Module | Description |
10+
|--------|-------------|
11+
| `ast` | Abstract syntax tree |
12+
| `cProfile` | Deterministic profiling |
13+
| `csv` | CSV file reading/writing |
14+
| `datetime` | Date and time types |
15+
| `email` | Email handling |
16+
| `filecmp` | File and directory comparison |
17+
| `fnmatch` | Unix filename pattern matching |
18+
| `glob` | Unix-style pathname expansion |
19+
| `imaplib` | IMAP4 protocol client |
20+
| `json` | JSON encoder/decoder |
21+
| `os` | OS interfaces |
22+
| `pathlib` | Object-oriented filesystem paths |
23+
| `platform` | Platform identification |
24+
| `profile` | Python profiler |
25+
| `re` | Regular expressions |
26+
| `shutil` | High-level file operations |
27+
| `socket` | Low-level networking |
28+
| `ssl` | TLS/SSL wrapper |
29+
| `subprocess` | Subprocess management (shimmed) |
30+
| `tempfile` | Temporary files and directories |
31+
32+
Other standard library modules (`collections`, `itertools`, `functools`, `hashlib`, `struct`, `threading`, `typing`, `urllib`, `xml`, `zipfile`, `tarfile`, `sqlite3`, etc.) are also available.
33+
34+
## Pre-imported packages (zero import cost)
35+
36+
These packages are imported during `pyhl setup` warmup. They are already in `sys.modules` when your code runs, so `import` is instant.
37+
38+
| Package | Import name |
39+
|---------|-------------|
40+
| beautifulsoup4 | `bs4` |
41+
| click | `click` |
42+
| cryptography | `cryptography` |
43+
| Jinja2 | `jinja2` |
44+
| lxml | `lxml` |
45+
| markdown-it-py | `markdown_it` |
46+
| numpy | `numpy` |
47+
| openpyxl | `openpyxl` |
48+
| pandas | `pandas` |
49+
| Pillow | `PIL` |
50+
| pydantic | `pydantic` |
51+
| pypdf | `pypdf` |
52+
| python-dateutil | `dateutil` |
53+
| python-docx | `docx` |
54+
| python-dotenv | `dotenv` |
55+
| python-pptx | `pptx` |
56+
| PyYAML | `yaml` |
57+
| tabulate | `tabulate` |
58+
| tenacity | `tenacity` |
59+
| tqdm | `tqdm` |
60+
61+
## Shipped packages (import cost on first use)
62+
63+
These packages are in the rootfs but not pre-imported. The first `import` pays the usual module load cost.
64+
65+
| Package | Import name |
66+
|---------|-------------|
67+
| aiohttp | `aiohttp` |
68+
| altair | `altair` |
69+
| APScheduler | `apscheduler` |
70+
| bandit | `bandit` |
71+
| bokeh | `bokeh` |
72+
| boto3 | `boto3` |
73+
| builtwith | `builtwith` |
74+
| celery | `celery` |
75+
| chardet | `chardet` |
76+
| charset-normalizer | `charset_normalizer` |
77+
| coverage | `coverage` |
78+
| distro | `distro` |
79+
| docx2txt | `docx2txt` |
80+
| duckdb | `duckdb` |
81+
82+
| exchange-calendars | `exchange_calendars` |
83+
| fabric | `fabric` |
84+
| Faker | `faker` |
85+
| fastapi | `fastapi` |
86+
| feedparser | `feedparser` |
87+
| fpdf2 | `fpdf` |
88+
| gensim | `gensim` |
89+
| gitpython | `git` |
90+
| google-api-python-client | `googleapiclient` |
91+
| hypercorn | `hypercorn` |
92+
| httpx | `httpx` |
93+
| hypothesis | `hypothesis` |
94+
| loguru | `loguru` |
95+
| markdown | `markdown` |
96+
| markdownify | `markdownify` |
97+
| mutagen | `mutagen` |
98+
| networkx | `networkx` |
99+
| nltk | `nltk` |
100+
| numpy-financial | `numpy_financial` |
101+
| odfpy | `odf` |
102+
| paramiko | `paramiko` |
103+
| pdfplumber | `pdfplumber` |
104+
| pdfrw | `pdfrw` |
105+
| pexpect | `pexpect` |
106+
| pipdeptree | `pipdeptree` |
107+
| platformdirs | `platformdirs` |
108+
| plotly | `plotly` |
109+
| polars | `polars` |
110+
| praw | `praw` |
111+
| pycountry | `pycountry` |
112+
| pydub | `pydub` |
113+
| pyflakes | `pyflakes` |
114+
| pygments | `pygments` |
115+
| pylint | `pylint` |
116+
| PyPDF2 | `PyPDF2` |
117+
| pytest | `pytest` |
118+
| pytest-asyncio | `pytest_asyncio` |
119+
| pytest-cov | `pytest_cov` |
120+
| pyxlsb | `pyxlsb` |
121+
| qrcode | `qrcode` |
122+
| radon | `radon` |
123+
| rapidfuzz | `rapidfuzz` |
124+
| rarfile | `rarfile` |
125+
| reportlab | `reportlab` |
126+
| requests | `requests` |
127+
| rope | `rope` |
128+
| ruff | `ruff` |
129+
| schedule | `schedule` |
130+
| scikit-learn | `sklearn` |
131+
| scipy | `scipy` |
132+
| scrapy | `scrapy` |
133+
| send2trash | `send2trash` |
134+
| slack-sdk | `slack_sdk` |
135+
| srt | `srt` |
136+
| statsmodels | `statsmodels` |
137+
| svgwrite | `svgwrite` |
138+
| sympy | `sympy` |
139+
| textblob | `textblob` |
140+
| trafilatura | `trafilatura` |
141+
| tweepy | `tweepy` |
142+
| typer | `typer` |
143+
| typing-extensions | `typing_extensions` |
144+
| uvicorn | `uvicorn` |
145+
| vulture | `vulture` |
146+
| watchdog | `watchdog` |
147+
| websockets | `websockets` |
148+
| wordcloud | `wordcloud` |
149+
| xlrd | `xlrd` |
150+
| xlsxwriter | `xlsxwriter` |

‎examples/python-agent-driver/Dockerfile‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,38 @@ ARG BASE=ghcr.io/hyperlight-dev/hyperlight-unikraft/python-base:latest
1010

1111
# Stage 1: Python deps (same as python-agent).
1212
FROM python:3.12-slim AS deps
13-
RUN pip install --target=/deps --no-cache-dir \
13+
RUN pip install --target=/deps --no-cache-dir --prefer-binary \
1414
tqdm pyyaml jinja2 beautifulsoup4 tabulate click tenacity \
1515
python-dotenv pypdf openpyxl markdown-it-py pydantic pillow \
16-
lxml cryptography python-dateutil numpy pandas
16+
lxml cryptography python-dateutil numpy pandas \
17+
python-docx python-pptx \
18+
chardet charset-normalizer \
19+
requests httpx aiohttp \
20+
feedparser markdown markdownify \
21+
Faker pycountry \
22+
loguru schedule send2trash \
23+
duckdb polars \
24+
xlrd xlsxwriter pyxlsb odfpy \
25+
pdfplumber pdfrw PyPDF2 \
26+
qrcode svgwrite \
27+
rapidfuzz \
28+
networkx sympy \
29+
pydub srt mutagen \
30+
plotly altair bokeh \
31+
statsmodels scikit-learn scipy \
32+
wordcloud \
33+
nltk textblob gensim \
34+
fastapi uvicorn hypercorn \
35+
typer pygments platformdirs distro \
36+
pytest pytest-cov pytest-asyncio coverage hypothesis \
37+
ruff pylint pyflakes bandit vulture radon rope \
38+
websockets \
39+
fpdf2 reportlab \
40+
APScheduler celery \
41+
numpy-financial docx2txt pipdeptree watchdog rarfile \
42+
boto3 google-api-python-client slack-sdk praw tweepy \
43+
scrapy trafilatura builtwith exchange-calendars \
44+
paramiko fabric pexpect gitpython
1745
RUN set -eux; \
1846
find /deps -maxdepth 1 -type d -name '*.dist-info' -exec rm -rf {} + 2>/dev/null || true; \
1947
find /deps \( -type d -name tests -o -type d -name test \) -exec rm -rf {} + 2>/dev/null || true; \
@@ -41,6 +69,7 @@ RUN PY_INC=$(python3.12 -c 'import sysconfig; print(sysconfig.get_path("include"
4169
FROM ${BASE} AS rootfs
4270
COPY --from=deps /deps /usr/local/lib/python3.12/site-packages
4371
COPY --from=driver-build /src/hl_pydriver /bin/hl_pydriver
72+
COPY pydoc_stub.py /usr/local/lib/python3.12/pydoc.py
4473

4574
# Stage 4: pack CPIO.
4675
FROM alpine:3.20 AS cpio

‎examples/python-agent-driver/Justfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ export DOCKER_BUILDKIT := "0"
1414

1515
kernel := ".unikraft/build/python-agent-driver-hyperlight_hyperlight-x86_64"
1616
initrd := "python-agent-driver-initrd.cpio"
17-
memory := "1Gi"
17+
memory := "2560Mi"
1818
image := "python-agent-driver-hyperlight"
1919
script := "../python-agent/agent.py"
2020

‎examples/python-agent-driver/hl_pydriver.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,8 @@ static void py_initialize_once(void)
252252
" 'numpy', 'pandas', 'pydantic', 'yaml', 'jinja2',"
253253
" 'bs4', 'tabulate', 'click', 'tenacity', 'tqdm',"
254254
" 'openpyxl', 'pypdf', 'markdown_it', 'PIL', 'lxml',"
255-
" 'cryptography', 'dateutil', 'dotenv'):\n"
255+
" 'cryptography', 'dateutil', 'dotenv',"
256+
" 'docx', 'pptx'):\n"
256257
" try:\n"
257258
" importlib.import_module(_mod)\n"
258259
" except Exception as _e:\n"

‎examples/python-agent-driver/pydoc_stub.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
"""Minimal pydoc stub — provides getdoc() for pyarrow's vendored docscrape."""
2+
3+
def getdoc(obj):
4+
try:
5+
doc = obj.__doc__
6+
except AttributeError:
7+
return ''
8+
if not doc:
9+
return ''
10+
import re
11+
return re.sub('^ *\n', '', doc.rstrip())

‎host/src/bin/pydriver_run.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ fn main() -> Result<()> {
4444
let t_evolve = Instant::now();
4545
let mut sandbox = Sandbox::builder(&kernel)
4646
.initrd_file(&initrd)
47-
.heap_size(2 * 1024 * 1024 * 1024)
47+
.heap_size(5 * 512 * 1024 * 1024)
4848
.build()?;
4949
eprintln!(
5050
"[timing] evolve={:.1}ms",

‎host/src/bin/pyhl.rs‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,8 @@ const PREIMPORTED_MODULES: &[&str] = &[
8989
"cryptography",
9090
"dateutil",
9191
"dotenv",
92+
"docx",
93+
"pptx",
9294
];
9395

9496
/// Build the long-about blurb shown by `pyhl --help`. Lists the
@@ -114,9 +116,23 @@ fn long_about() -> String {
114116
}
115117
s.push_str(
116118
"\n\n\
117-
Other third-party packages shipped in the rootfs still work — \
118-
they just pay the usual import cost on first access. Packages \
119-
not in the rootfs will raise ModuleNotFoundError.",
119+
Additional packages shipped in the rootfs (pay import cost on \
120+
first use): aiohttp, altair, APScheduler, bandit, bokeh, \
121+
boto3, builtwith, celery, chardet, charset-normalizer, \
122+
coverage, distro, docx2txt, duckdb, \
123+
exchange-calendars, fabric, Faker, fastapi, feedparser, \
124+
fpdf2, gensim, gitpython, google-api-python-client, \
125+
hypercorn, httpx, hypothesis, loguru, markdown, markdownify, \
126+
mutagen, networkx, nltk, numpy-financial, odfpy, paramiko, \
127+
pdfplumber, pdfrw, pexpect, pipdeptree, platformdirs, plotly, \
128+
polars, praw, pycountry, pydub, pyflakes, pygments, pylint, \
129+
PyPDF2, pytest, pytest-asyncio, pytest-cov, pyxlsb, qrcode, \
130+
radon, rapidfuzz, rarfile, reportlab, requests, rope, ruff, \
131+
schedule, scikit-learn, scipy, scrapy, send2trash, slack-sdk, \
132+
srt, statsmodels, svgwrite, sympy, textblob, trafilatura, \
133+
tweepy, typer, uvicorn, vulture, watchdog, websockets, \
134+
wordcloud, xlrd, xlsxwriter.\n\n\
135+
Packages not in the rootfs will raise ModuleNotFoundError.",
120136
);
121137
s
122138
}
@@ -435,7 +451,7 @@ fn cmd_setup(args: SetupArgs) -> Result<()> {
435451
{
436452
let mut builder = Sandbox::builder(&dst_kernel)
437453
.initrd_file(&dst_initrd)
438-
.heap_size(3 * 512 * 1024 * 1024);
454+
.heap_size(5 * 512 * 1024 * 1024);
439455
for p in &setup_preopens {
440456
builder = builder.preopen(p.clone());
441457
}
@@ -577,18 +593,12 @@ fn cmd_run(args: RunArgs) -> Result<()> {
577593
listen_ports.is_some(),
578594
)?;
579595

580-
let initrd = home.join(INITRD_FILE);
581-
582596
let t_load = Instant::now();
583-
let initrd_ref = if initrd.is_file() {
584-
Some(initrd.as_path())
585-
} else {
586-
None
587-
};
597+
let initrd = home.join(INITRD_FILE);
588598
let mut sandbox = Sandbox::from_snapshot_file_configured(
589599
&snapshot,
590600
&run_preopens,
591-
initrd_ref,
601+
Some(initrd.as_path()),
592602
network.as_ref(),
593603
listen_ports.as_ref(),
594604
)?;

‎host/src/pyhl.rs‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ pub fn install(opts: &InstallOptions<'_>) -> Result<InstallReport> {
197197
{
198198
let mut builder = Sandbox::builder(&dst_kernel)
199199
.initrd_file(&dst_initrd)
200-
.heap_size(3 * 512 * 1024 * 1024);
200+
.heap_size(5 * 512 * 1024 * 1024);
201201
for p in opts.mounts {
202202
builder = builder.preopen(p.clone());
203203
}
@@ -275,15 +275,10 @@ impl Runtime {
275275
);
276276
}
277277
let initrd = home.join(INITRD_FILE);
278-
let initrd_ref = if initrd.is_file() {
279-
Some(initrd.as_path())
280-
} else {
281-
None
282-
};
283278
let sandbox = Sandbox::from_snapshot_file_configured(
284279
&snap,
285280
mounts,
286-
initrd_ref,
281+
Some(initrd.as_path()),
287282
network,
288283
listen_ports,
289284
)?;

0 commit comments

Comments
 (0)