feat: expand python-agent-driver to 103 pip packages

Add 85 new pip packages to the rootfs, bringing the total from 18 to
103. Six of the new packages (docx, pptx, chardet, charset_normalizer,
requests, httpx) are pre-imported during warmup for zero-cost access.

Bump heap from 1.5 GB to 2.5 GB across pyhl CLI, pyhl library,
pydriver_run, and the Justfile to accommodate the larger initrd.

Signed-off-by: danbugs <danilochiarlone@gmail.com>

Author: danbugs

examples/python-agent-driver/Dockerfile

@@ -13,7 +13,35 @@ FROM python:3.12-slim AS deps
 RUN pip install --target=/deps --no-cache-dir \
     tqdm pyyaml jinja2 beautifulsoup4 tabulate click tenacity \
     python-dotenv pypdf openpyxl markdown-it-py pydantic pillow \
-    lxml cryptography python-dateutil numpy pandas
+    lxml cryptography python-dateutil numpy pandas \
+    python-docx python-pptx \
+    chardet charset-normalizer \
+    requests httpx aiohttp \
+    feedparser markdown markdownify \
+    Faker pycountry \
+    loguru schedule send2trash \
+    duckdb polars \
+    xlrd xlsxwriter pyxlsb odfpy \
+    pdfplumber pdfrw PyPDF2 \
+    qrcode svgwrite \
+    rapidfuzz \
+    networkx sympy \
+    pydub srt mutagen \
+    plotly altair bokeh \
+    statsmodels scikit-learn scipy \
+    wordcloud \
+    nltk textblob gensim \
+    fastapi uvicorn hypercorn \
+    typer pygments platformdirs distro \
+    pytest pytest-cov pytest-asyncio coverage hypothesis \
+    ruff pylint pyflakes bandit vulture radon rope \
+    websockets \
+    fpdf2 reportlab \
+    APScheduler celery \
+    numpy-financial docx2txt pipdeptree watchdog rarfile \
+    boto3 google-api-python-client slack-sdk praw tweepy \
+    scrapy trafilatura builtwith edgartools exchange-calendars \
+    paramiko fabric pexpect gitpython
 RUN set -eux; \
     find /deps -maxdepth 1 -type d -name '*.dist-info' -exec rm -rf {} + 2>/dev/null || true; \
     find /deps \( -type d -name tests -o -type d -name test \) -exec rm -rf {} + 2>/dev/null || true; \

examples/python-agent-driver/Justfile

@@ -14,7 +14,7 @@ export DOCKER_BUILDKIT := "0"
 
 kernel   := ".unikraft/build/python-agent-driver-hyperlight_hyperlight-x86_64"
 initrd   := "python-agent-driver-initrd.cpio"
-memory   := "1Gi"
+memory   := "2560Mi"
 image    := "python-agent-driver-hyperlight"
 script   := "../python-agent/agent.py"
 

examples/python-agent-driver/hl_pydriver.c

@@ -252,7 +252,9 @@ static void py_initialize_once(void)
 		"    'numpy', 'pandas', 'pydantic', 'yaml', 'jinja2',"
 		"    'bs4', 'tabulate', 'click', 'tenacity', 'tqdm',"
 		"    'openpyxl', 'pypdf', 'markdown_it', 'PIL', 'lxml',"
-		"    'cryptography', 'dateutil', 'dotenv'):\n"
+		"    'cryptography', 'dateutil', 'dotenv',"
+		"    'docx', 'pptx', 'chardet', 'charset_normalizer',"
+		"    'requests', 'httpx'):\n"
 		"  try:\n"
 		"    importlib.import_module(_mod)\n"
 		"  except Exception as _e:\n"

host/src/bin/pydriver_run.rs

@@ -44,7 +44,7 @@ fn main() -> Result<()> {
     let t_evolve = Instant::now();
     let mut sandbox = Sandbox::builder(&kernel)
         .initrd_file(&initrd)
-        .heap_size(2 * 1024 * 1024 * 1024)
+        .heap_size(5 * 512 * 1024 * 1024)
         .build()?;
     eprintln!(
         "[timing] evolve={:.1}ms",

host/src/bin/pyhl.rs

@@ -89,6 +89,12 @@ const PREIMPORTED_MODULES: &[&str] = &[
     "cryptography",
     "dateutil",
     "dotenv",
+    "docx",
+    "pptx",
+    "chardet",
+    "charset_normalizer",
+    "requests",
+    "httpx",
 ];
 
 /// Build the long-about blurb shown by `pyhl --help`. Lists the
@@ -114,9 +120,22 @@ fn long_about() -> String {
     }
     s.push_str(
         "\n\n\
-         Other third-party packages shipped in the rootfs still work — \
-         they just pay the usual import cost on first access. Packages \
-         not in the rootfs will raise ModuleNotFoundError.",
+         Additional packages shipped in the rootfs (pay import cost on \
+         first use): aiohttp, altair, APScheduler, bandit, bokeh, \
+         boto3, builtwith, celery, coverage, distro, docx2txt, duckdb, \
+         edgartools, exchange-calendars, fabric, Faker, fastapi, \
+         feedparser, fpdf2, gensim, gitpython, \
+         google-api-python-client, hypercorn, hypothesis, loguru, \
+         markdown, markdownify, mutagen, networkx, nltk, \
+         numpy-financial, odfpy, paramiko, pdfplumber, pdfrw, pexpect, \
+         pipdeptree, platformdirs, plotly, polars, praw, pycountry, \
+         pydub, pyflakes, pygments, pylint, PyPDF2, pytest, \
+         pytest-asyncio, pytest-cov, pyxlsb, qrcode, radon, rapidfuzz, \
+         rarfile, reportlab, rope, ruff, schedule, scikit-learn, scipy, \
+         scrapy, send2trash, slack-sdk, srt, statsmodels, svgwrite, \
+         sympy, textblob, trafilatura, tweepy, typer, uvicorn, vulture, \
+         watchdog, websockets, wordcloud, xlrd, xlsxwriter.\n\n\
+         Packages not in the rootfs will raise ModuleNotFoundError.",
     );
     s
 }
@@ -435,7 +454,7 @@ fn cmd_setup(args: SetupArgs) -> Result<()> {
     {
         let mut builder = Sandbox::builder(&dst_kernel)
             .initrd_file(&dst_initrd)
-            .heap_size(3 * 512 * 1024 * 1024);
+            .heap_size(5 * 512 * 1024 * 1024);
         for p in &setup_preopens {
             builder = builder.preopen(p.clone());
         }

host/src/pyhl.rs

@@ -197,7 +197,7 @@ pub fn install(opts: &InstallOptions<'_>) -> Result<InstallReport> {
     {
         let mut builder = Sandbox::builder(&dst_kernel)
             .initrd_file(&dst_initrd)
-            .heap_size(3 * 512 * 1024 * 1024);
+            .heap_size(5 * 512 * 1024 * 1024);
         for p in opts.mounts {
             builder = builder.preopen(p.clone());
         }