diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index b60764a45d..e73e1c7516 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -182,8 +182,8 @@ jobs: with: ecosystem-ci-project: ${{ matrix.project.name }} - - name: Install pnpm - uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4 + - name: Setup utoo + uses: utooland/setup-utoo@3a51006d0b66afcc32d1b9177a4b200b74f4a8cb # main - name: Set up Node.js uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 @@ -191,19 +191,23 @@ jobs: node-version: ${{ matrix.project.node-version }} - name: Install dependencies - run: pnpm install --no-frozen-lockfile + run: ut install --from pnpm - name: Build all packages - env: - # publint pack defaults to npm (main CI env has no pnpm); in E2E we - # already have pnpm installed and npm pack against pnpm's symlinked - # node_modules is ~10x slower, so prefer pnpm pack here - PUBLINT_PACK: pnpm - run: pnpm build + run: ut run build - name: Pack packages into tgz + # `ut pm-pack` resolves `workspace:` / `catalog:` protocols only on + # utoo >= 1.1, which is not yet on the `latest` tag that setup-utoo + # installs. Install 1.1.x into an isolated prefix and point pack-all.mjs + # at it via UT_BIN, so the install/build steps above keep using the + # repo-default utoo. pack-all.mjs generates `.utoo.toml` and injects a + # temporary `workspaces` field (both required by pm-pack), then writes + # one tgz per publishable package to the repo root for patch-project.ts. run: | - pnpm -r pack + UT_PREFIX="$(mktemp -d)" + npm install --prefix "$UT_PREFIX" --no-save --no-fund --no-audit utoo@1.1.1 + UT_BIN="$UT_PREFIX/node_modules/.bin/ut" node ecosystem-ci/pack-all.mjs - name: Override dependencies from tgz in ${{ matrix.project.name }} working-directory: ecosystem-ci/${{ matrix.project.name }} diff --git a/ecosystem-ci/pack-all.mjs b/ecosystem-ci/pack-all.mjs new file mode 100644 index 0000000000..e4d67f56c0 --- /dev/null +++ b/ecosystem-ci/pack-all.mjs @@ -0,0 +1,167 @@ +#!/usr/bin/env node +/** + * Pack every publishable workspace package into a tgz at the repo root using + * utoo's `ut pm-pack`, as a drop-in replacement for `pnpm -r pack` in the + * ecosystem-ci (E2E) workflow. + * + * Why this exists (utoo >= 1.1 quirks): + * - `ut pm-pack` resolves `workspace:` deps via the npm-style `workspaces` + * field in the root package.json -- NOT from pnpm-workspace.yaml. So we + * temporarily inject `workspaces` (mirrored from pnpm-workspace.yaml) for the + * duration of packing, then restore package.json. + * - `ut pm-pack` resolves `catalog:` deps from `.utoo.toml`, NOT from + * pnpm-workspace.yaml. We generate `.utoo.toml` from pnpm-workspace.yaml. + * - `ut pm-pack` does NOT apply `publishConfig` overrides the way npm/pnpm do. + * egg packages keep dev `exports` pointing at `src/*.ts` and override them to + * `dist/*.js` via `publishConfig.exports`, so we apply `publishConfig` onto + * each manifest before packing (then restore it), otherwise the tarballs ship + * `src` exports and downstream installs fail with MODULE_NOT_FOUND. + * - `ut pm-pack ` writes the tgz INTO the package dir (no + * --pack-destination). patch-project.ts expects all tgz at the repo root with + * npm-standard names, so we move them up. + * + * The `ut` binary can be overridden with UT_BIN (used by local validation to + * point at a pinned utoo version). The repository tree is restored afterward. + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import { glob } from 'node:fs/promises'; +import path from 'node:path'; + +import yaml from 'js-yaml'; + +import { generateUtooToml } from '../scripts/gen-utoo-catalog.mjs'; + +const rootDir = path.join(import.meta.dirname, '..'); +const UT_BIN = process.env.UT_BIN || (process.platform === 'win32' ? 'ut.cmd' : 'ut'); + +// publishConfig keys that are manifest fields consumers read, which npm/pnpm +// copy onto the published manifest at publish time. Use an allowlist so +// publish-only keys (access, tag, registry, ignore, ...) never leak into the +// packed package.json. Mirrors pnpm's publish-time overridable field set. +const PUBLISHABLE_MANIFEST_FIELDS = new Set([ + 'bin', + 'main', + 'exports', + 'types', + 'typings', + 'module', + 'browser', + 'esnext', + 'es2015', + 'unpkg', + 'umd:main', +]); + +const ws = yaml.load(fs.readFileSync(path.join(rootDir, 'pnpm-workspace.yaml'), 'utf8')); + +// Read a file, returning null when it does not exist (avoids a TOCTOU +// existsSync check before reading). +function readFileOrNull(filePath) { + try { + return fs.readFileSync(filePath, 'utf8'); + } catch (err) { + if (err.code === 'ENOENT') return null; + throw err; + } +} + +// Discover publishable packages exactly like patch-project.ts: glob each +// workspace pattern for package.json, skip private / nameless packages. +async function discoverPackages() { + const packages = []; + for (const pattern of ws.packages) { + for await (const entry of glob(`${pattern}/package.json`, { cwd: rootDir })) { + const pkgJsonPath = path.join(rootDir, entry); + try { + const pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8')); + if (pkgJson.private || !pkgJson.name) continue; + packages.push({ name: pkgJson.name, dir: path.dirname(entry), version: pkgJson.version }); + } catch { + console.warn(`Warning: could not read ${pkgJsonPath}`); + } + } + } + return packages; +} + +// npm-standard tarball name, matches patch-project.ts's expectation. +function tgzName(name, version) { + return `${name.replace('@', '').replace('/', '-')}-${version}.tgz`; +} + +// Apply publishConfig manifest overrides (e.g. exports -> dist) the way +// npm/pnpm do at publish time, skipping publish-control-only keys. +function applyPublishConfig(manifest) { + const pc = manifest.publishConfig; + if (!pc) return manifest; + for (const [key, value] of Object.entries(pc)) { + if (PUBLISHABLE_MANIFEST_FIELDS.has(key)) manifest[key] = value; + } + return manifest; +} + +async function main() { + const pkgJsonPath = path.join(rootDir, 'package.json'); + const originalPkgJson = fs.readFileSync(pkgJsonPath, 'utf8'); + const utooTomlPath = path.join(rootDir, '.utoo.toml'); + const originalUtooToml = readFileOrNull(utooTomlPath); + + // package.json files we mutated and still owe a restore (path -> original). + const pendingRestores = new Map(); + + try { + // 1. Generate .utoo.toml so pm-pack can resolve catalog:/catalog:. + fs.writeFileSync(utooTomlPath, generateUtooToml(rootDir)); + + // 2. Inject npm-style `workspaces` so pm-pack can discover workspace pkgs. + const pkgJson = JSON.parse(originalPkgJson); + pkgJson.workspaces = ws.packages; + fs.writeFileSync(pkgJsonPath, JSON.stringify(pkgJson, null, 2) + '\n'); + + const packages = await discoverPackages(); + console.log(`📦 Packing ${packages.length} packages with ${UT_BIN} pm-pack`); + + for (const pkg of packages) { + // 3. Apply publishConfig (exports -> dist, etc.) before packing so the + // tarball ships the published manifest, then restore the source file. + const manifestPath = path.join(rootDir, pkg.dir, 'package.json'); + const originalManifest = fs.readFileSync(manifestPath, 'utf8'); + pendingRestores.set(manifestPath, originalManifest); + const manifest = applyPublishConfig(JSON.parse(originalManifest)); + fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n'); + + execFileSync(UT_BIN, ['pm-pack', pkg.dir], { cwd: rootDir, stdio: 'inherit' }); + + fs.writeFileSync(manifestPath, originalManifest); + pendingRestores.delete(manifestPath); + + const file = tgzName(pkg.name, pkg.version); + const from = path.join(rootDir, pkg.dir, file); + const to = path.join(rootDir, file); + try { + fs.renameSync(from, to); + } catch (err) { + throw new Error(`Expected tarball not found: ${from}`, { cause: err }); + } + console.log(` -> ${file}`); + } + console.log(`✅ Packed ${packages.length} tarballs into ${rootDir}`); + } finally { + // Restore everything we touched. + fs.writeFileSync(pkgJsonPath, originalPkgJson); + for (const [manifestPath, original] of pendingRestores) { + fs.writeFileSync(manifestPath, original); + } + if (originalUtooToml === null) { + fs.rmSync(utooTomlPath, { force: true }); + } else { + fs.writeFileSync(utooTomlPath, originalUtooToml); + } + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/gen-utoo-catalog.mjs b/scripts/gen-utoo-catalog.mjs new file mode 100644 index 0000000000..7e691163d9 --- /dev/null +++ b/scripts/gen-utoo-catalog.mjs @@ -0,0 +1,81 @@ +#!/usr/bin/env node +/** + * Generate `.utoo.toml` from `pnpm-workspace.yaml`. + * + * `ut pm-pack` (utoo >= 1.1) resolves `catalog:` / `catalog:` protocols + * from `.utoo.toml`, NOT from `pnpm-workspace.yaml`. To keep + * `pnpm-workspace.yaml` the single source of truth, we generate the TOML mirror + * on demand instead of committing a hand-maintained copy. + * + * pnpm `catalog:` map -> `[catalog]` + * pnpm `catalogs.` -> `[catalogs.]` + * + * Usage: + * node scripts/gen-utoo-catalog.mjs # writes /.utoo.toml + * node scripts/gen-utoo-catalog.mjs --print # print to stdout, write nothing + */ +import fs from 'node:fs'; +import path from 'node:path'; + +import yaml from 'js-yaml'; + +// TOML keys that are not bare-key-safe (A-Za-z0-9_-) must be quoted. +const BARE_KEY = /^[A-Za-z0-9_-]+$/; +function tomlKey(name) { + return BARE_KEY.test(name) ? name : JSON.stringify(name); +} +function tomlValue(version) { + // version specs are always strings; JSON.stringify gives a valid TOML basic string + return JSON.stringify(String(version)); +} + +function renderCatalogTable(entries) { + return Object.keys(entries) + .sort() + .map((name) => `${tomlKey(name)} = ${tomlValue(entries[name])}`) + .join('\n'); +} + +export function generateUtooToml(rootDir = process.cwd()) { + const wsPath = path.join(rootDir, 'pnpm-workspace.yaml'); + const ws = yaml.load(fs.readFileSync(wsPath, 'utf8')) ?? {}; + + const blocks = [ + '# AUTO-GENERATED from pnpm-workspace.yaml by scripts/gen-utoo-catalog.mjs', + '# Do not edit by hand. Source of truth is pnpm-workspace.yaml.', + ]; + + if (ws.catalog && Object.keys(ws.catalog).length > 0) { + blocks.push(`[catalog]\n${renderCatalogTable(ws.catalog)}`); + } + + if (ws.catalogs && Object.keys(ws.catalogs).length > 0) { + for (const catalogName of Object.keys(ws.catalogs).sort()) { + const entries = ws.catalogs[catalogName]; + if (entries && Object.keys(entries).length > 0) { + blocks.push(`[catalogs.${tomlKey(catalogName)}]\n${renderCatalogTable(entries)}`); + } + } + } + + return blocks.join('\n\n') + '\n'; +} + +function main() { + const rootDir = process.cwd(); + const toml = generateUtooToml(rootDir); + if (process.argv.includes('--print')) { + process.stdout.write(toml); + return; + } + const out = path.join(rootDir, '.utoo.toml'); + fs.writeFileSync(out, toml); + console.log(`Wrote ${out} (${toml.split('\n').length} lines)`); +} + +// Run main() only when invoked directly. Compare resolved paths (not +// `file://${argv[1]}`) so the check also holds on Windows, where argv[1] uses +// backslashes while import.meta.url is a forward-slash file URL. +if (process.argv[1] && import.meta.filename === path.resolve(process.argv[1])) { + main(); +}