Skip to content

Commit e286683

Browse files
github-actions[bot]Copilot
github-actions[bot]
and
Copilot
committed
[Autoloop: perf-comparison] Iteration 406: readSas benchmark (1k rows, 20 iters)
Add TS and Python benchmarks for readSas/pd.read_sas using in-memory SAS XPORT v5 files with 3 numeric columns and 1 character column. Run: https://github.com/githubnext/tsb/actions/runs/29583613301 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent cada908 commit e286683

2 files changed

Lines changed: 296 additions & 0 deletions

File tree

‎benchmarks/pandas/bench_read_sas.py‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
"""
2+
Benchmark: pd.read_sas — parse a 1,000-row SAS XPORT (XPT) file.
3+
Outputs JSON: {"function": "read_sas", "mean_ms": ..., "iterations": ..., "total_ms": ...}
4+
"""
5+
import json
6+
import time
7+
import io
8+
import struct
9+
import math
10+
import numpy as np
11+
import pandas as pd
12+
13+
ROWS = 1_000
14+
WARMUP = 3
15+
ITERATIONS = 20
16+
17+
# ── IBM 370 double encoder ────────────────────────────────────────────────────
18+
19+
def ibm_encode(val: float) -> bytes:
20+
if val == 0.0:
21+
return b"\x00" * 8
22+
if not math.isfinite(val):
23+
return b"\x2e" + b"\x00" * 7
24+
sign = 1 if val < 0 else 0
25+
abs_val = abs(val)
26+
exp = 0
27+
mant = abs_val
28+
while mant >= 1.0:
29+
mant /= 16
30+
exp += 1
31+
while mant < 1.0 / 16 and mant > 0:
32+
mant *= 16
33+
exp -= 1
34+
mant_int = round(mant * 2**56)
35+
out = bytearray(8)
36+
out[0] = (sign << 7) | ((exp + 64) & 0x7f)
37+
for i in range(1, 8):
38+
out[i] = (mant_int >> ((7 - i) * 8)) & 0xff
39+
return bytes(out)
40+
41+
# ── Minimal XPORT v5 builder ─────────────────────────────────────────────────
42+
43+
def build_xpt(num_vars, char_vars, rows_data):
44+
RECORD = 80
45+
46+
def pad80(s):
47+
return s.ljust(RECORD).encode("ascii")[:RECORD]
48+
49+
def write_u16(val):
50+
return struct.pack(">H", val)
51+
52+
def write_u32(val):
53+
return struct.pack(">I", val)
54+
55+
# Compute variable metadata
56+
metas = []
57+
pos = 0
58+
for name in num_vars:
59+
metas.append({"type": 1, "name": name, "len": 8, "pos": pos})
60+
pos += 8
61+
for name, length in char_vars:
62+
metas.append({"type": 2, "name": name, "len": length, "pos": pos})
63+
pos += length
64+
row_len = pos
65+
66+
chunks = bytearray()
67+
68+
# Library header (5 × 80 bytes)
69+
chunks += pad80("HEADER RECORD*******LIBRARY HEADER RECORD!!!!!!!000000000000000000000000000000 ")
70+
chunks += pad80("SAS SAS SASLIB 6.06 ASCII")
71+
chunks += pad80("20240101")
72+
chunks += pad80("")
73+
chunks += pad80("")
74+
75+
# Member header (3 × 80 bytes)
76+
chunks += pad80("HEADER RECORD*******MEMBER HEADER RECORD!!!!!!!000000000000000000000000000001600000000140 ")
77+
chunks += pad80("SAS BENCH SASDATA 6.06 ASCII")
78+
chunks += pad80("")
79+
80+
# Namestr header
81+
nvar = len(metas)
82+
chunks += pad80(f"HEADER RECORD*******NAMESTR HEADER RECORD!!!!!!!{nvar:06d}00000000000000000000 ")
83+
84+
# Namestr records (140 bytes each)
85+
ns_buf = bytearray(nvar * 140)
86+
for i, m in enumerate(metas):
87+
off = i * 140
88+
ns_buf[off:off+2] = write_u16(m["type"])
89+
ns_buf[off+2:off+4] = write_u16(140)
90+
name_bytes = m["name"].encode("ascii").ljust(8)[:8]
91+
ns_buf[off+4:off+12] = name_bytes
92+
ns_buf[off+52:off+54] = write_u16(m["len"])
93+
ns_buf[off+84:off+88] = write_u32(m["pos"])
94+
padded_ns = math.ceil(len(ns_buf) / RECORD) * RECORD
95+
ns_buf_padded = ns_buf.ljust(padded_ns, b"\x00")
96+
chunks += ns_buf_padded
97+
98+
# Obs header
99+
chunks += pad80("HEADER RECORD*******OBS HEADER RECORD!!!!!!!000000000000000000000000000000 ")
100+
101+
# Observations
102+
padded_row_len = math.ceil(row_len / RECORD) * RECORD
103+
obs_buf = bytearray(len(rows_data) * padded_row_len)
104+
for r, row in enumerate(rows_data):
105+
base = r * padded_row_len
106+
for m in metas:
107+
val = row.get(m["name"])
108+
if m["type"] == 1:
109+
encoded = ibm_encode(float(val) if val is not None else 0.0)
110+
obs_buf[base + m["pos"]:base + m["pos"] + 8] = encoded
111+
else:
112+
s = str(val) if val is not None else ""
113+
b = s.encode("ascii")[:m["len"]].ljust(m["len"], b" ")
114+
obs_buf[base + m["pos"]:base + m["pos"] + m["len"]] = b
115+
116+
chunks += obs_buf
117+
return bytes(chunks)
118+
119+
120+
# ── Build dataset ─────────────────────────────────────────────────────────────
121+
122+
rows_data = [
123+
{"id": float(i), "value": i * 1.5, "score": math.sin(i * 0.01), "label": f"item_{i % 100}"}
124+
for i in range(ROWS)
125+
]
126+
127+
xpt_bytes = build_xpt(
128+
["id", "value", "score"],
129+
[("label", 12)],
130+
rows_data,
131+
)
132+
133+
# ── Benchmark ─────────────────────────────────────────────────────────────────
134+
135+
for _ in range(WARMUP):
136+
pd.read_sas(io.BytesIO(xpt_bytes), format="xport")
137+
138+
start = time.perf_counter()
139+
for _ in range(ITERATIONS):
140+
pd.read_sas(io.BytesIO(xpt_bytes), format="xport")
141+
total = (time.perf_counter() - start) * 1000
142+
143+
print(json.dumps({
144+
"function": "read_sas",
145+
"mean_ms": total / ITERATIONS,
146+
"iterations": ITERATIONS,
147+
"total_ms": total,
148+
}))

‎benchmarks/tsb/bench_read_sas.ts‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/**
2+
* Benchmark: readSas — parse a 1,000-row SAS XPORT (XPT) file.
3+
* Outputs JSON: {"function": "read_sas", "mean_ms": ..., "iterations": ..., "total_ms": ...}
4+
*/
5+
import { readSas } from "../../src/index.ts";
6+
7+
// ─── IBM 370 floating-point encoder ──────────────────────────────────────────
8+
9+
function ibmEncode(val: number): Uint8Array {
10+
const out = new Uint8Array(8);
11+
if (val === 0) return out;
12+
if (!Number.isFinite(val)) { out[0] = 0x2e; return out; }
13+
const sign = val < 0 ? 1 : 0;
14+
const abs = Math.abs(val);
15+
let exp = 0;
16+
let mant = abs;
17+
while (mant >= 1) { mant /= 16; exp++; }
18+
while (mant < 1 / 16 && mant > 0) { mant *= 16; exp--; }
19+
const mantInt = BigInt(Math.round(mant * 2 ** 56));
20+
out[0] = (sign << 7) | ((exp + 64) & 0x7f);
21+
for (let i = 1; i <= 7; i++) {
22+
out[i] = Number((mantInt >> BigInt((7 - i) * 8)) & 0xffn);
23+
}
24+
return out;
25+
}
26+
27+
// ─── Minimal XPORT v5 builder ────────────────────────────────────────────────
28+
29+
function buildXpt(
30+
numVars: readonly string[],
31+
charVars: readonly { name: string; len: number }[],
32+
rows: readonly Readonly<Record<string, number | string>>[],
33+
): Uint8Array {
34+
const RECORD = 80;
35+
36+
function encodeAscii(s: string, maxLen: number): Uint8Array {
37+
const buf = new Uint8Array(maxLen);
38+
for (let i = 0; i < Math.min(s.length, maxLen); i++) buf[i] = s.charCodeAt(i) & 0x7f;
39+
return buf;
40+
}
41+
function padTo80(s: string): Uint8Array { return encodeAscii(s.padEnd(RECORD, " "), RECORD); }
42+
function writeU16(b: Uint8Array, o: number, v: number) { b[o] = (v >> 8) & 0xff; b[o + 1] = v & 0xff; }
43+
function writeU32(b: Uint8Array, o: number, v: number) {
44+
b[o] = (v >> 24) & 0xff; b[o + 1] = (v >> 16) & 0xff; b[o + 2] = (v >> 8) & 0xff; b[o + 3] = v & 0xff;
45+
}
46+
47+
type Meta = { type: 1 | 2; name: string; len: number; pos: number };
48+
const metas: Meta[] = [];
49+
let pos = 0;
50+
for (const name of numVars) { metas.push({ type: 1, name, len: 8, pos }); pos += 8; }
51+
for (const { name, len } of charVars) { metas.push({ type: 2, name, len, pos }); pos += len; }
52+
const rowLen = pos;
53+
54+
const chunks: Uint8Array[] = [];
55+
56+
// Library header (5 × 80 bytes)
57+
chunks.push(padTo80("HEADER RECORD*******LIBRARY HEADER RECORD!!!!!!!000000000000000000000000000000 "));
58+
chunks.push(padTo80("SAS SAS SASLIB 6.06 ASCII"));
59+
chunks.push(padTo80("20240101"));
60+
chunks.push(padTo80(""));
61+
chunks.push(padTo80(""));
62+
63+
// Member header (3 × 80 bytes)
64+
chunks.push(padTo80("HEADER RECORD*******MEMBER HEADER RECORD!!!!!!!000000000000000000000000000001600000000140 "));
65+
chunks.push(padTo80("SAS BENCH SASDATA 6.06 ASCII"));
66+
chunks.push(padTo80(""));
67+
68+
// Namestr header
69+
const nvar = metas.length;
70+
chunks.push(padTo80(`HEADER RECORD*******NAMESTR HEADER RECORD!!!!!!!${String(nvar).padStart(6, "0")}00000000000000000000 `));
71+
72+
// Namestr records (140 bytes each)
73+
const nsBuf = new Uint8Array(nvar * 140);
74+
for (let i = 0; i < metas.length; i++) {
75+
const m = metas[i]!;
76+
const off = i * 140;
77+
writeU16(nsBuf, off, m.type);
78+
writeU16(nsBuf, off + 2, 140);
79+
nsBuf.set(encodeAscii(m.name, 8), off + 4);
80+
writeU16(nsBuf, off + 52, m.len);
81+
writeU32(nsBuf, off + 84, m.pos);
82+
}
83+
const nsPadded = Math.ceil(nsBuf.length / RECORD) * RECORD;
84+
const nsPaddedBuf = new Uint8Array(nsPadded);
85+
nsPaddedBuf.set(nsBuf);
86+
chunks.push(nsPaddedBuf);
87+
88+
// Obs header
89+
chunks.push(padTo80("HEADER RECORD*******OBS HEADER RECORD!!!!!!!000000000000000000000000000000 "));
90+
91+
// Observations
92+
const paddedRowLen = Math.ceil(rowLen / RECORD) * RECORD;
93+
const obsBuf = new Uint8Array(rows.length * paddedRowLen);
94+
for (let r = 0; r < rows.length; r++) {
95+
const base = r * paddedRowLen;
96+
const row = rows[r]!;
97+
for (const m of metas) {
98+
const val = row[m.name];
99+
if (m.type === 1) {
100+
const encoded = ibmEncode(typeof val === "number" ? val : 0);
101+
obsBuf.set(encoded, base + m.pos);
102+
} else {
103+
const s = typeof val === "string" ? val : "";
104+
obsBuf.set(encodeAscii(s, m.len), base + m.pos);
105+
}
106+
}
107+
}
108+
chunks.push(obsBuf);
109+
110+
let total = 0;
111+
for (const c of chunks) total += c.length;
112+
const out = new Uint8Array(total);
113+
let off = 0;
114+
for (const c of chunks) { out.set(c, off); off += c.length; }
115+
return out;
116+
}
117+
118+
// ─── Build dataset ────────────────────────────────────────────────────────────
119+
120+
const ROWS = 1_000;
121+
const WARMUP = 3;
122+
const ITERATIONS = 20;
123+
124+
const rows: Readonly<Record<string, number | string>>[] = Array.from({ length: ROWS }, (_, i) => ({
125+
id: i,
126+
value: i * 1.5,
127+
score: Math.sin(i * 0.01),
128+
label: `item_${i % 100}`,
129+
}));
130+
131+
const xpt = buildXpt(["id", "value", "score"], [{ name: "label", len: 12 }], rows);
132+
133+
// ─── Benchmark ────────────────────────────────────────────────────────────────
134+
135+
for (let i = 0; i < WARMUP; i++) readSas(xpt);
136+
137+
const start = performance.now();
138+
for (let i = 0; i < ITERATIONS; i++) readSas(xpt);
139+
const total = performance.now() - start;
140+
141+
console.log(
142+
JSON.stringify({
143+
function: "read_sas",
144+
mean_ms: total / ITERATIONS,
145+
iterations: ITERATIONS,
146+
total_ms: total,
147+
}),
148+
);

0 commit comments

Comments
 (0)