Skip to content

Commit 2e94d0b

Browse files
committed
feat: implement cross-version stdlib smoke test for Python in CI
- Added a new Python script `smoke_stdlib.py` to perform cross-version compatibility checks for major C extensions. - Updated GitLab CI and GitHub Actions workflows to include the new smoke test, ensuring it runs against both Python 2.7 and Python 3.x. - Refined existing CI scripts to remove outdated import checks and streamline the testing process for Python packages.
1 parent e012290 commit 2e94d0b

5 files changed

Lines changed: 251 additions & 14 deletions

File tree

.github/workflows/build.yml

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,15 @@ jobs: {
2626
"run": "docker build -f ./${{ matrix.version }}/${{ matrix.architecture }}/Dockerfile -t release-${{ github.ref_name }}:${{ matrix.version }}-${{ matrix.architecture }} .",
2727
},
2828
{
29-
"name": "Smoke test python and pip in final image",
30-
"run": "IMPORTS='import ssl, sqlite3, ctypes'; \
31-
if [ '${{ matrix.version }}' != '2.7' ]; then IMPORTS=\"$IMPORTS, lzma\"; fi; \
32-
docker run --rm release-${{ github.ref_name }}:${{ matrix.version }}-${{ matrix.architecture }} /opt/python/bin/python -c \"$IMPORTS; print('ok')\" && \
33-
docker run --rm release-${{ github.ref_name }}:${{ matrix.version }}-${{ matrix.architecture }} /opt/python/bin/pip --version",
29+
"name": "pip version",
30+
"run": "docker run --rm release-${{ github.ref_name }}:${{ matrix.version }}-${{ matrix.architecture }} /opt/python/bin/pip --version",
31+
},
32+
{
33+
"name": "Cross-version stdlib smoke",
34+
"run": "docker run --rm \
35+
-v \"${{ github.workspace }}/ci/smoke_stdlib.py:/tmp/smoke_stdlib.py:ro\" \
36+
release-${{ github.ref_name }}:${{ matrix.version }}-${{ matrix.architecture }} \
37+
/opt/python/bin/python /tmp/smoke_stdlib.py",
3438
},
3539
{
3640
"name": "psutil compatibility smoke test (Py3 only)",

.gitlab-ci.yml

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,17 @@ build:
1818
- (sudo apt-get update -y && sudo apt-get install -y gzip) || true
1919
script:
2020
- docker build -f ./${version}/${architecture}/Dockerfile -t release-${PACKAGE_NAME}:${version}-${architecture} . 1>${version}-${architecture}.log 2>&1
21-
# lzma is Py3-only stdlib, so strip it from the import list for 2.7.
22-
# Everything else is common across 2/3.
23-
- |
24-
IMPORTS='import ssl, sqlite3, ctypes'
25-
if [ "$version" != "2.7" ]; then IMPORTS="$IMPORTS, lzma"; fi
26-
docker run --rm release-${PACKAGE_NAME}:${version}-${architecture} \
27-
/opt/python/bin/python -c "$IMPORTS; print('ok')"
2821
- docker run --rm release-${PACKAGE_NAME}:${version}-${architecture} /opt/python/bin/pip --version
22+
# Cross-version stdlib smoke. Exercises ssl (OpenSSL), hashlib digests,
23+
# sqlite3, zlib, bz2, lzma (Py3-only; self-skips on 2.7), ctypes, curses,
24+
# readline, socket, subprocess, threading, json. Exits non-zero if any
25+
# section fails, so CI catches regressions in the shipped libc / deplib
26+
# chain without us having to keep the inline check in sync by hand.
27+
- |
28+
docker run --rm \
29+
-v "${CI_PROJECT_DIR}/ci/smoke_stdlib.py:/tmp/smoke_stdlib.py:ro" \
30+
release-${PACKAGE_NAME}:${version}-${architecture} \
31+
/opt/python/bin/python /tmp/smoke_stdlib.py
2932
# End-to-end compatibility check: install a native-extension wheel from
3033
# PyPI (exercises the musllinux tag hook) and run a broad surface-area
3134
# smoke test against the shipped libc / /proc / subprocess plumbing.

ci/smoke_stdlib.py

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
#!/usr/bin/env python
2+
# -*- coding: utf-8 -*-
3+
"""Cross-version stdlib smoke test for standalone-python.
4+
5+
Exercises the major C extensions and their backing libraries. Compatible
6+
with both Python 2.7 and Python 3.x so CI can run the same script against
7+
every matrix cell.
8+
9+
Exit status: 0 if every section either succeeded or was explicitly
10+
skipped (e.g. `lzma` on 2.7). Non-zero if any unexpected exception.
11+
12+
Run:
13+
./python /tmp/smoke_stdlib.py
14+
"""
15+
16+
from __future__ import print_function
17+
18+
import os
19+
import sys
20+
import traceback
21+
22+
23+
_IS_PY3 = sys.version_info[0] >= 3
24+
SECTIONS = []
25+
26+
27+
def section(label):
28+
def decorator(fn):
29+
SECTIONS.append((label, fn))
30+
return fn
31+
return decorator
32+
33+
34+
# ---------------------------------------------------------------------------
35+
# Sections
36+
# ---------------------------------------------------------------------------
37+
38+
39+
@section("version / platform")
40+
def s_version():
41+
print(" " + sys.version.splitlines()[0])
42+
print(" executable : " + sys.executable)
43+
print(" platform : " + sys.platform + " / " + os.uname()[4])
44+
45+
46+
@section("ssl (OpenSSL)")
47+
def s_ssl():
48+
import ssl
49+
print(" OPENSSL : " + ssl.OPENSSL_VERSION)
50+
ctx = ssl.create_default_context()
51+
assert hasattr(ctx, "check_hostname")
52+
# If the build is going to fail spectacularly against OpenSSL, it'll
53+
# usually do it in the context constructor (missing symbols at load).
54+
55+
56+
@section("hashlib (md5/sha1/sha256/sha512)")
57+
def s_hashlib():
58+
import hashlib
59+
# Known-answer tests against the empty string so a silently-wrong
60+
# backend is detectable.
61+
kats = {
62+
"md5": "d41d8cd98f00b204e9800998ecf8427e",
63+
"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
64+
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
65+
"sha512": ("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce"
66+
"47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"),
67+
}
68+
for algo, expected in kats.items():
69+
h = hashlib.new(algo)
70+
got = h.hexdigest()
71+
assert got == expected, algo + " wrong: " + got + " != " + expected
72+
avail = sorted(hashlib.algorithms_available)
73+
print(" backed : " + str(len(avail)) + " algorithms via _hashlib")
74+
75+
76+
@section("sqlite3")
77+
def s_sqlite():
78+
import sqlite3
79+
conn = sqlite3.connect(":memory:")
80+
try:
81+
cur = conn.cursor()
82+
cur.execute("CREATE TABLE t (k TEXT, v INTEGER)")
83+
cur.executemany("INSERT INTO t VALUES (?, ?)", [("a", 1), ("b", 2), ("c", 3)])
84+
cur.execute("SELECT SUM(v) FROM t")
85+
(total,) = cur.fetchone()
86+
assert total == 6
87+
finally:
88+
conn.close()
89+
print(" library : " + sqlite3.sqlite_version)
90+
91+
92+
@section("zlib")
93+
def s_zlib():
94+
import zlib
95+
data = b"standalone-python " * 100
96+
c = zlib.compress(data, 9)
97+
d = zlib.decompress(c)
98+
assert d == data
99+
print(" library : " + zlib.ZLIB_VERSION +
100+
", ratio " + str(len(c)) + "/" + str(len(data)))
101+
102+
103+
@section("bz2")
104+
def s_bz2():
105+
import bz2
106+
data = b"The quick brown fox " * 50
107+
c = bz2.compress(data)
108+
d = bz2.decompress(c)
109+
assert d == data
110+
111+
112+
@section("lzma (Py3-only)")
113+
def s_lzma():
114+
if not _IS_PY3:
115+
print(" skipped (Py2 has no stdlib _lzma module)")
116+
return
117+
import lzma
118+
data = b"lzma smoke " * 100
119+
c = lzma.compress(data)
120+
d = lzma.decompress(c)
121+
assert d == data
122+
123+
124+
@section("ctypes (libc via dlopen)")
125+
def s_ctypes():
126+
import ctypes
127+
libc = ctypes.CDLL(None) # NULL handle = main program / already-loaded libc
128+
libc.getpid.restype = ctypes.c_int
129+
assert libc.getpid() == os.getpid()
130+
# strlen is a good signal for reaching libc proper
131+
libc.strlen.argtypes = [ctypes.c_char_p]
132+
libc.strlen.restype = ctypes.c_size_t
133+
assert libc.strlen(b"standalone") == 10
134+
135+
136+
@section("curses")
137+
def s_curses():
138+
# Don't call initscr() — that needs a real tty. Just confirm the C
139+
# extension loads and exposes the expected symbols.
140+
import curses
141+
assert hasattr(curses, "initscr")
142+
assert hasattr(curses, "color_pair")
143+
assert hasattr(curses, "ACS_HLINE")
144+
# panel is a separate extension (_curses_panel); check both.
145+
import curses.panel
146+
assert hasattr(curses.panel, "new_panel")
147+
148+
149+
@section("readline")
150+
def s_readline():
151+
import readline
152+
assert hasattr(readline, "parse_and_bind")
153+
assert hasattr(readline, "get_history_length")
154+
155+
156+
@section("socket")
157+
def s_socket():
158+
import socket
159+
# Don't require network — just exercise the parsing layers.
160+
addrs = socket.getaddrinfo("127.0.0.1", 80, 0, socket.SOCK_STREAM)
161+
assert addrs, "no addrinfo for 127.0.0.1:80"
162+
# IPv6 parsing
163+
packed = socket.inet_pton(socket.AF_INET6, "::1")
164+
assert len(packed) == 16
165+
166+
167+
@section("subprocess (spawn child, check rc + output)")
168+
def s_subprocess():
169+
import subprocess
170+
out = subprocess.check_output(
171+
[sys.executable, "-c", "import sys; sys.stdout.write('42')"]
172+
)
173+
assert b"42" in out, "unexpected child output: " + repr(out)
174+
175+
176+
@section("threading (basic Thread run/join)")
177+
def s_threading():
178+
import threading
179+
results = []
180+
t = threading.Thread(target=lambda: results.append("ok"))
181+
t.start()
182+
t.join(timeout=2)
183+
assert not t.is_alive(), "thread still alive after join"
184+
assert results == ["ok"]
185+
186+
187+
@section("json")
188+
def s_json():
189+
import json
190+
s = json.dumps({"a": [1, 2, 3], "b": "standålone"})
191+
back = json.loads(s)
192+
assert back["a"] == [1, 2, 3]
193+
194+
195+
# ---------------------------------------------------------------------------
196+
# Driver
197+
# ---------------------------------------------------------------------------
198+
199+
200+
def main():
201+
failed = []
202+
for label, fn in SECTIONS:
203+
print(label + ":")
204+
try:
205+
fn()
206+
except Exception:
207+
failed.append(label)
208+
traceback.print_exc()
209+
210+
print()
211+
print("=" * 50)
212+
total = len(SECTIONS)
213+
passed = total - len(failed)
214+
print("summary: {0}/{1} passed".format(passed, total))
215+
if failed:
216+
print("FAILED: " + ", ".join(failed))
217+
return 1
218+
return 0
219+
220+
221+
if __name__ == "__main__":
222+
sys.exit(main())

common/build/deplib/build_python.sh

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,12 @@ tar -xvf Python-$PYTHON_VERSION.tar.xz && cd Python-$PYTHON_VERSION
1111
export EXTRA_CFLAGS="-DTHREAD_STACK_SIZE=0x100000"
1212
export LDFLAGS="${LDFLAGS:--Wl},--strip-all"
1313

14-
export LOCAL_INCLUDES="-I/opt/shared_libraries/include/ncurses"
14+
# Our ncurses is built with --enable-widec (ncurses 6.x default), which
15+
# installs headers under `include/ncursesw/` (not `include/ncurses/`) and
16+
# exposes libncursesw.so. Without this include path Python's setup.py
17+
# can't find `curses.h` and silently skips building `_curses` /
18+
# `_curses_panel` / `readline` wide-char bits.
19+
export LOCAL_INCLUDES="-I/opt/shared_libraries/include/ncursesw"
1520
export LOCAL_INCLUDES="${LOCAL_INCLUDES} -I/opt/shared_libraries/include/" # some wired packages include ncurses.h or ncurses/ncurses.h
1621

1722
export CFLAGS="${CFLAGS} ${LOCAL_INCLUDES}"

common/build/deplib/build_python_py2.sh

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@ export LDFLAGS="${LDFLAGS:--Wl},--strip-all"
3535

3636
# 2.7's setup.py scans CPPFLAGS/LDFLAGS to find OpenSSL / sqlite / ncurses /
3737
# etc. Point those at the shipped prefix so OpenSSL 1.1.1w is discovered.
38-
export LOCAL_INCLUDES="-I/opt/shared_libraries/include/ncurses"
38+
# Our ncurses is built with --enable-widec (ncurses 6.x default), so
39+
# headers live under `include/ncursesw/` and libraries are `libncursesw*`.
40+
# Without the ncursesw include setup.py silently skips building `_curses`.
41+
export LOCAL_INCLUDES="-I/opt/shared_libraries/include/ncursesw"
3942
export LOCAL_INCLUDES="${LOCAL_INCLUDES} -I/opt/shared_libraries/include/"
4043
export CFLAGS="${CFLAGS} ${LOCAL_INCLUDES}"
4144
export CPPFLAGS="${CPPFLAGS} ${LOCAL_INCLUDES}"

0 commit comments

Comments
 (0)