Skip to content

Commit 03f867b

Browse files
mverzilliaztec-bot
authored andcommitted
fix: include sqlite binary in its npm package (#23039)
We correctly .gitignore prebuilt sqlite artifacts since we fetch and verify them on build, however, this is inconvenient for users of our packages, since it leaves the burden of re-vendoring up to them. This PR introduces an .npmignore file to the sqlite npm package, which by convention overshadows the .gitignore file when npm produces the package tarball. It also includes a verification script to make sure there's no loose ends in the process
1 parent 9e17f5b commit 03f867b

5 files changed

Lines changed: 127 additions & 3 deletions

File tree

yarn-project/sqlite3mc-wasm/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818
"build": "yarn clean && ../scripts/tsc.sh",
1919
"clean": "rm -rf ./dest .tsbuildinfo",
2020
"test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}",
21-
"build:dev": "../scripts/tsc.sh --watch"
21+
"build:dev": "../scripts/tsc.sh --watch",
22+
"prepublishOnly": "./scripts/verify-pack.sh"
2223
},
2324
"devDependencies": {
2425
"@jest/globals": "^30.0.0",

yarn-project/sqlite3mc-wasm/scripts/vendor.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ PKG_ROOT=$(cd "$SCRIPT_DIR/.." && pwd)
2929

3030
LOCAL_DMTS="$PKG_ROOT/vendor/jswasm/sqlite3-bundler-friendly.d.mts"
3131
LOCAL_GITIGNORE="$PKG_ROOT/vendor/jswasm/.gitignore"
32+
LOCAL_NPMIGNORE="$PKG_ROOT/vendor/jswasm/.npmignore"
3233
SHA256SUMS="$PKG_ROOT/vendor/jswasm/SHA256SUMS"
3334
PIN_FILE="$SCRIPT_DIR/vendor.pin"
3435

@@ -100,6 +101,7 @@ fi
100101
# Preserve files that aren't part of the upstream release across re-vendoring:
101102
# - sqlite3-bundler-friendly.d.mts: locally-authored TypeScript declaration
102103
# - .gitignore: allowlist that keeps upstream artifacts untracked
104+
# - .npmignore: shadows .gitignore so npm publish keeps the artifacts
103105
DMTS_BACKUP=""
104106
if [[ -f "$LOCAL_DMTS" ]]; then
105107
DMTS_BACKUP=$(mktemp)
@@ -110,6 +112,11 @@ if [[ -f "$LOCAL_GITIGNORE" ]]; then
110112
GITIGNORE_BACKUP=$(mktemp)
111113
cp "$LOCAL_GITIGNORE" "$GITIGNORE_BACKUP"
112114
fi
115+
NPMIGNORE_BACKUP=""
116+
if [[ -f "$LOCAL_NPMIGNORE" ]]; then
117+
NPMIGNORE_BACKUP=$(mktemp)
118+
cp "$LOCAL_NPMIGNORE" "$NPMIGNORE_BACKUP"
119+
fi
113120

114121
echo "==> Replacing vendor/jswasm/ with pristine upstream files"
115122
rm -rf "$PKG_ROOT/vendor/jswasm"
@@ -127,6 +134,11 @@ if [[ -n "$GITIGNORE_BACKUP" ]]; then
127134
rm "$GITIGNORE_BACKUP"
128135
echo "==> Restored .gitignore"
129136
fi
137+
if [[ -n "$NPMIGNORE_BACKUP" ]]; then
138+
cp "$NPMIGNORE_BACKUP" "$LOCAL_NPMIGNORE"
139+
rm "$NPMIGNORE_BACKUP"
140+
echo "==> Restored .npmignore"
141+
fi
130142

131143
echo "==> Generating vendor/jswasm/SHA256SUMS"
132144
(cd "$PKG_ROOT/vendor/jswasm" && sha256sum -- * 2>/dev/null | sort -k2 > SHA256SUMS)
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#!/usr/bin/env bash
2+
# Verify that the npm tarball this package would publish includes every
3+
# vendored sqlite3mc artifact listed in vendor/jswasm/SHA256SUMS.
4+
#
5+
# Background: npm honors any `.gitignore` it finds inside directories listed
6+
# in the `files` allowlist. The `.gitignore` in vendor/jswasm/ excludes
7+
# everything except an allowlist (SHA256SUMS, the locally-authored .d.mts,
8+
# and itself), which inadvertently strips the WASM/MJS artifacts from the
9+
# published tarball even though they're present on disk after vendor.sh ran.
10+
#
11+
# vendor/jswasm/.npmignore shadows that .gitignore for npm pack purposes.
12+
# This script is the guard that catches any future regression. For example:
13+
# someone deletes the .npmignore, or a new vendored file is added but not
14+
# captured by whatever inclusion mechanism is in place. Wired into
15+
# `prepublishOnly` so a broken tarball aborts publish before upload.
16+
#
17+
# Exit codes: 0 = all expected files present, 1 = missing files, 2 = setup
18+
# error (no SHA256SUMS, npm pack failed, etc.).
19+
20+
set -euo pipefail
21+
22+
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
23+
PKG_ROOT=$(cd "$SCRIPT_DIR/.." && pwd)
24+
SHA256SUMS="$PKG_ROOT/vendor/jswasm/SHA256SUMS"
25+
26+
if [[ ! -f "$SHA256SUMS" ]]; then
27+
echo "verify-pack: SHA256SUMS not found at $SHA256SUMS" >&2
28+
echo " Run scripts/vendor.sh first to populate vendor/jswasm/." >&2
29+
exit 2
30+
fi
31+
32+
# Parse SHA256SUMS for the list of files that MUST appear in the tarball.
33+
# Each line is "<sha256> <filename>". We want the filename, prefixed with
34+
# the package-relative path that npm uses inside the tarball.
35+
expected=()
36+
while IFS= read -r line; do
37+
fname=$(awk '{print $2}' <<<"$line")
38+
[[ -n "$fname" ]] || continue
39+
expected+=("vendor/jswasm/$fname")
40+
done < "$SHA256SUMS"
41+
42+
if [[ ${#expected[@]} -eq 0 ]]; then
43+
echo "verify-pack: SHA256SUMS contained no entries, nothing to verify" >&2
44+
exit 2
45+
fi
46+
47+
WORK_DIR=$(mktemp -d)
48+
trap 'rm -rf "$WORK_DIR"' EXIT
49+
50+
# Build the same tarball `npm publish` would upload, then list its contents.
51+
# `--dry-run --json` output format varies across npm versions; running real
52+
# `npm pack` and untarring is stable across npm 7+.
53+
echo "==> Running npm pack to build a candidate tarball"
54+
(cd "$PKG_ROOT" && npm pack --pack-destination "$WORK_DIR" >/dev/null)
55+
TARBALL=$(find "$WORK_DIR" -maxdepth 1 -name '*.tgz' | head -n 1)
56+
if [[ -z "$TARBALL" ]]; then
57+
echo "verify-pack: npm pack produced no tarball in $WORK_DIR" >&2
58+
exit 2
59+
fi
60+
61+
# Tarball entries are prefixed with "package/" — strip for comparison.
62+
tar tzf "$TARBALL" | sed 's|^package/||' > "$WORK_DIR/listing"
63+
64+
missing=0
65+
for f in "${expected[@]}"; do
66+
if ! grep -Fxq "$f" "$WORK_DIR/listing"; then
67+
echo "verify-pack: missing from tarball: $f" >&2
68+
missing=1
69+
fi
70+
done
71+
72+
if [[ "$missing" -eq 1 ]]; then
73+
cat >&2 <<EOF
74+
75+
verify-pack: one or more vendored sqlite3mc artifacts were filtered out of
76+
the npm tarball. Most likely cause: vendor/jswasm/.npmignore is missing,
77+
letting the sibling .gitignore strip the WASM/MJS artifacts at pack time.
78+
79+
To diagnose:
80+
cd $PKG_ROOT
81+
npm pack --dry-run 2>&1 | grep vendor/jswasm
82+
83+
Tarball file listing (for reference): $WORK_DIR/listing
84+
EOF
85+
# Don't auto-clean WORK_DIR on failure so the operator can inspect.
86+
trap - EXIT
87+
echo "verify-pack: leaving $WORK_DIR for inspection" >&2
88+
exit 1
89+
fi
90+
91+
echo "verify-pack: all ${#expected[@]} vendored files present in tarball"
Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
# Upstream sqlite3mc-wasm release artifacts are fetched at build time by
22
# scripts/vendor.sh (driven by scripts/vendor.pin). Don't commit them.
33
#
4-
# Allowlist: keep this .gitignore, the integrity manifest, and our
5-
# locally-authored TypeScript declaration tracked in git.
4+
# Allowlist: keep this .gitignore, the .npmignore that shadows it for npm
5+
# pack, the integrity manifest, and our locally-authored TypeScript
6+
# declaration tracked in git.
67
*
78
!.gitignore
9+
!.npmignore
810
!SHA256SUMS
911
!sqlite3-bundler-friendly.d.mts
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Empty allowlist override for `npm publish`.
2+
#
3+
# The sibling .gitignore is structured to keep upstream-fetched binary
4+
# artifacts (sqlite3mc WASM/MJS files) out of git: they're regenerated by
5+
# scripts/vendor.sh on every checkout/build.
6+
#
7+
# But: when `files` in package.json lists `vendor/`, npm's tarball builder
8+
# also honors any nested .gitignore, and would strip those same artifacts
9+
# from the published tarball, leaving downstream consumers with an
10+
# unbuildable package (`dest/index.js` re-exports from
11+
# `../vendor/jswasm/sqlite3-bundler-friendly.mjs`).
12+
#
13+
# An .npmignore in a directory shadows the sibling .gitignore for npm pack.
14+
# Empty contents = "include everything in this directory."
15+
#
16+
# Guard: scripts/verify-pack.sh runs at prepublishOnly and asserts every
17+
# file in SHA256SUMS appears in the produced tarball. If this file gets
18+
# deleted by accident, that check fires before the broken tarball ships.

0 commit comments

Comments
 (0)