Skip to content

Commit 72175a3

Browse files
author
Robins Kiste
committed
Fix fast-load import maps and rebuild SWC wasm
1 parent 4b80cfa commit 72175a3

8 files changed

Lines changed: 207 additions & 35 deletions

File tree

‎flatn/bun-install.js‎

Lines changed: 48 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { execSync, spawnSync } from 'child_process';
1+
import { execSync, spawn, spawnSync } from 'child_process';
22
import fs from 'fs';
33
import path from 'path';
44
import { gitSpecFromVersion } from './flatn-cjs.js';
@@ -77,16 +77,7 @@ export async function bunInstall (bunPath, livelyDirs, destDir, projectRoot, ver
7777
);
7878

7979
// 4. Run bun install
80-
const result = spawnSync(bunPath, ['install', '--no-progress'], {
81-
cwd: bunWorkDir,
82-
stdio: verbose ? 'inherit' : 'pipe',
83-
env: { ...process.env }
84-
});
85-
86-
if (result.status !== 0) {
87-
const stderr = result.stderr ? result.stderr.toString() : '';
88-
throw new Error(`bun install failed (exit ${result.status}): ${stderr}`);
89-
}
80+
await runBunInstall(bunPath, bunWorkDir, verbose);
9081
// bun install completed
9182

9283
// 5. Build git spec map from original dependency version strings
@@ -137,6 +128,52 @@ export async function bunInstall (bunPath, livelyDirs, destDir, projectRoot, ver
137128
return { newPackages };
138129
}
139130

131+
async function runBunInstall (bunPath, bunWorkDir, verbose) {
132+
console.log(' Running bun install...');
133+
134+
const child = spawn(bunPath, ['install', '--no-progress'], {
135+
cwd: bunWorkDir,
136+
stdio: verbose ? 'inherit' : ['ignore', 'pipe', 'pipe'],
137+
env: { ...process.env }
138+
});
139+
140+
let stdout = '';
141+
let stderr = '';
142+
let lastOutputAt = Date.now();
143+
const startedAt = Date.now();
144+
145+
if (!verbose) {
146+
child.stdout?.on('data', chunk => {
147+
stdout += chunk.toString();
148+
lastOutputAt = Date.now();
149+
});
150+
child.stderr?.on('data', chunk => {
151+
stderr += chunk.toString();
152+
lastOutputAt = Date.now();
153+
});
154+
}
155+
156+
const heartbeat = !verbose && setInterval(() => {
157+
const elapsedSec = Math.round((Date.now() - startedAt) / 1000);
158+
const quietSec = Math.round((Date.now() - lastOutputAt) / 1000);
159+
console.log(` bun install still running... ${elapsedSec}s elapsed, ${quietSec}s since last output`);
160+
}, 10000);
161+
162+
const result = await new Promise((resolve, reject) => {
163+
child.on('error', reject);
164+
child.on('close', (code, signal) => resolve({ code, signal }));
165+
});
166+
167+
if (heartbeat) clearInterval(heartbeat);
168+
169+
if (result.code !== 0) {
170+
const output = [stderr.trim(), stdout.trim()].filter(Boolean).join('\n');
171+
throw new Error(
172+
`bun install failed (${result.signal ? `signal ${result.signal}` : `exit ${result.code}`}): ${output}`
173+
);
174+
}
175+
}
176+
140177
function buildGitDepMap (aggregatedDeps, bunWorkDir, bunPath) {
141178
// Build a map of package name -> gitSpec using the ORIGINAL dependency version
142179
// strings. This is critical because PackageSpec.matches() computes its gitSpec

‎lively.freezer/src/util/bootstrap.js‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -185,8 +185,10 @@ async function shallowReloadModulesIfNeeded (modulesToCheck, moduleHashes, R) {
185185
let key = modId;
186186
let currMod;
187187
if (key === '@empty') continue;
188-
if (key.startsWith('esm://')) continue; // do not revive esm modules
189-
if (modHash !== moduleHashes['/' + key]) {
188+
if (key.startsWith('esm://') || key.startsWith('http://') || key.startsWith('https://')) continue; // do not revive CDN modules
189+
const serverHash = moduleHashes['/' + key];
190+
if (serverHash == null) continue; // module is not part of the server hash map
191+
if (modHash !== serverHash) {
190192
console.log('reviving', modId);
191193
currMod = lively.modules.module(modId);
192194
try {
@@ -364,7 +366,8 @@ function bootstrapLivelySystem (progress, fastLoad = query.fastLoad !== false ||
364366
const keysAfter = obj.keys(R.registry);
365367
if (keysBefore.length < keysAfter.length) {
366368
// detect modules to be reloaded
367-
const modulesToUpdate = arr.withoutAll(keysAfter, keysBefore).filter(id => !id.startsWith('esm://'));
369+
const modulesToUpdate = arr.withoutAll(keysAfter, keysBefore)
370+
.filter(id => !id.startsWith('esm://') && !id.startsWith('http://') && !id.startsWith('https://'));
368371
for (const mod of modulesToUpdate) {
369372
System.set(System.decanonicalize(mod), System.newModule(R.exportsOf(mod)));
370373
const m = lively.modules.module(mod);

‎lively.freezer/swc-plugin/lively_swc_plugin.wasm‎

1.33 MB
Binary file not shown.

‎lively.ide/package.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,9 @@
4747
},
4848
"systemjs": {
4949
"map": {
50-
"highlight.js/lib/core": "https://ga.jspm.io/npm:highlight.js@11.11.1/es/core.js",
51-
"highlight.js/lib/languages/javascript": "https://ga.jspm.io/npm:highlight.js@11.11.1/es/languages/javascript.js",
52-
"highlight.js/lib/languages/shell": "https://ga.jspm.io/npm:highlight.js@11.11.1/es/languages/shell.js"
50+
"highlight.js/lib/core": "esm://ga.jspm.io/npm:highlight.js@11.11.1/es/core.js",
51+
"highlight.js/lib/languages/javascript": "esm://ga.jspm.io/npm:highlight.js@11.11.1/es/languages/javascript.js",
52+
"highlight.js/lib/languages/shell": "esm://ga.jspm.io/npm:highlight.js@11.11.1/es/languages/shell.js"
5353
}
5454
}
5555
}

‎lively.installer/install.js‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,15 @@ const spinner = {
2121
frames: ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'],
2222
idx: 0, timer: null, text: '', baseText: '', active: false,
2323
_origLog: console.log, _origWarn: console.warn, _origError: console.error,
24+
_clearLine () {
25+
if (!process.stdout.isTTY) return;
26+
if (typeof process.stdout.clearLine === 'function' && typeof process.stdout.cursorTo === 'function') {
27+
process.stdout.clearLine(0);
28+
process.stdout.cursorTo(0);
29+
return;
30+
}
31+
process.stdout.write('\r\x1b[K');
32+
},
2433
start (text) {
2534
if (this.text === text && this.active) return;
2635
if (this.active) this._completeLine();
@@ -38,29 +47,31 @@ const spinner = {
3847
},
3948
render () {
4049
const frame = this.frames[this.idx++ % this.frames.length];
41-
process.stdout.write(`\r ${frame} ${this.text}\x1b[K`);
50+
this._clearLine();
51+
process.stdout.write(` ${frame} ${this.text}`);
4252
},
4353
_completeLine () {
4454
if (this.timer) { clearInterval(this.timer); this.timer = null; }
4555
if (process.stdout.isTTY && this.baseText) {
46-
process.stdout.write(`\r \x1b[32m✓\x1b[0m ${this.baseText}\x1b[K\n`);
56+
this._clearLine();
57+
process.stdout.write(` \x1b[32m✓\x1b[0m ${this.baseText}\n`);
4758
}
4859
},
4960
_hookConsole () {
5061
if (console.log === this._wrappedLog) return;
5162
const self = this;
5263
this._wrappedLog = function (...args) {
53-
if (self.active && process.stdout.isTTY) process.stdout.write('\r\x1b[K');
64+
if (self.active && process.stdout.isTTY) self._clearLine();
5465
self._origLog.apply(console, args);
5566
if (self.active && process.stdout.isTTY) self.render();
5667
};
5768
this._wrappedWarn = function (...args) {
58-
if (self.active && process.stdout.isTTY) process.stdout.write('\r\x1b[K');
69+
if (self.active && process.stdout.isTTY) self._clearLine();
5970
self._origWarn.apply(console, args);
6071
if (self.active && process.stdout.isTTY) self.render();
6172
};
6273
this._wrappedError = function (...args) {
63-
if (self.active && process.stdout.isTTY) process.stdout.write('\r\x1b[K');
74+
if (self.active && process.stdout.isTTY) self._clearLine();
6475
self._origError.apply(console, args);
6576
if (self.active && process.stdout.isTTY) self.render();
6677
};

‎lively.morphic/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
"systemjs": {
4747
"main": "index.js",
4848
"map": {
49-
"yoga-layout/load": "https://ga.jspm.io/npm:yoga-layout@3.2.1/dist/src/load.js"
49+
"yoga-layout/load": "esm://ga.jspm.io/npm:yoga-layout@3.2.1/dist/src/load.js"
5050
}
5151
},
5252
"scripts": {

‎lively.server/plugins/dav.js‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,13 @@ export default class LivelyDAVPlugin {
105105
exclude: (res) => {
106106
return res.url.includes('lively.next-node_modules') ||
107107
res.url.includes('.module_cache') ||
108-
!res.url.startsWith(System.baseURL + 'lively') && !res.url.includes('esm_cache') ||
109-
res.isFile() && !res.url.endsWith('.js') && !res.url.endsWith('.cjs');
108+
!res.url.startsWith(System.baseURL + 'lively') &&
109+
!res.url.startsWith(System.baseURL + 'flatn') &&
110+
!res.url.includes('esm_cache') ||
111+
res.isFile() &&
112+
!res.url.endsWith('.js') &&
113+
!res.url.endsWith('.cjs') &&
114+
!res.url.endsWith('.mjs');
110115
}
111116
});
112117
for (let file of filesToHash) {

‎lively.server/plugins/lib-lookup.js‎

Lines changed: 126 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import { resource } from "lively.resources";
55
import { parseQuery } from "lively.resources";
66
import { arr, obj } from "lively.lang";
77
const Generator = System.get('@jspm_generator').default;
8+
const semver = System._nodeRequire('semver');
9+
let localDependerIndex;
810

911
// Deps that cannot be resolved via jspm.io CDN:
1012
// - native binary packages (platform-specific compiled addons)
@@ -33,6 +35,89 @@ function extractFailingPackage (errMsg) {
3335
return null;
3436
}
3537

38+
function extractImportingPackageScope (errMsg) {
39+
const importerMatch = errMsg.match(/imported from (https:\/\/ga\.jspm\.io\/npm:(?:@[^/]+\/)?[^@\s/]+@[^/]+\/)/);
40+
return importerMatch ? importerMatch[1] : null;
41+
}
42+
43+
function buildPackageUrl (name, version) {
44+
return `https://ga.jspm.io/npm:${name}@${version}/`;
45+
}
46+
47+
function toCachedScopeUrl (scopeUrl) {
48+
return scopeUrl.replace(/^https:\/\//, 'esm://');
49+
}
50+
51+
function toGeneratorScopeUrl (scopeUrl) {
52+
return scopeUrl.replace(/^esm:\/\//, 'https://');
53+
}
54+
55+
function indexPackageDependers (packageJson, index) {
56+
if (!packageJson?.name || !packageJson?.version) return;
57+
const deps = {
58+
...(packageJson.dependencies || {}),
59+
...(packageJson.peerDependencies || {}),
60+
...(packageJson.optionalDependencies || {})
61+
};
62+
for (const [depName, depRange] of Object.entries(deps)) {
63+
if (typeof depRange !== 'string') continue;
64+
(index[depName] ||= []).push({
65+
scopeUrl: buildPackageUrl(packageJson.name, packageJson.version),
66+
range: depRange
67+
});
68+
}
69+
}
70+
71+
function getLocalDependerIndex () {
72+
if (localDependerIndex) return localDependerIndex;
73+
74+
const index = {};
75+
const packageRoot = join(process.cwd(), 'lively.next-node_modules');
76+
if (!fs.existsSync(packageRoot)) {
77+
localDependerIndex = index;
78+
return index;
79+
}
80+
81+
const stack = [packageRoot];
82+
while (stack.length) {
83+
const dir = stack.pop();
84+
let entries;
85+
try {
86+
entries = fs.readdirSync(dir, { withFileTypes: true });
87+
} catch {
88+
continue;
89+
}
90+
for (const entry of entries) {
91+
const fullPath = join(dir, entry.name);
92+
if (entry.isDirectory()) {
93+
stack.push(fullPath);
94+
continue;
95+
}
96+
if (!entry.isFile() || entry.name !== 'package.json') continue;
97+
try {
98+
indexPackageDependers(JSON.parse(fs.readFileSync(fullPath, 'utf8')), index);
99+
} catch {}
100+
}
101+
}
102+
103+
localDependerIndex = index;
104+
return index;
105+
}
106+
107+
function findScopedPinTargets (pkgName, goodVersion, importerScope = null) {
108+
const scopedPins = new Set();
109+
if (importerScope) scopedPins.add(importerScope);
110+
const dependers = getLocalDependerIndex()[pkgName] || [];
111+
for (const { scopeUrl, range } of dependers) {
112+
try {
113+
if (semver.satisfies(goodVersion, range, { includePrerelease: true })) {
114+
scopedPins.add(scopeUrl);
115+
}
116+
} catch {}
117+
}
118+
return [...scopedPins];
119+
}
120+
36121
/**
37122
* Given a package name and failing version, find a nearby version that
38123
* actually exists on the jspm.io CDN by walking backwards from the failing
@@ -66,16 +151,26 @@ async function findAvailableCDNVersion (name, failingVersion) {
66151
return null;
67152
}
68153

69-
function createGenerator (inputMap, resolutions) {
70-
return new Generator({
154+
function createGenerator (inputMap, resolutions, scopedResolutions = {}) {
155+
const generator = new Generator({
71156
env: ["browser"],
72157
defaultProvider: 'jspm.io',
73158
inputMap,
74159
...(Object.keys(resolutions).length ? { resolutions } : {})
75160
});
161+
const installs = generator.traceMap.installer.installs;
162+
installs.secondary ||= {};
163+
installs.flattened ||= {};
164+
for (const [scopeUrl, pins] of Object.entries(scopedResolutions)) {
165+
const scope = installs.secondary[toGeneratorScopeUrl(scopeUrl)] ||= {};
166+
for (const [pkgName, version] of Object.entries(pins || {})) {
167+
scope[pkgName] = { installUrl: buildPackageUrl(pkgName, version) };
168+
}
169+
}
170+
return generator;
76171
}
77172

78-
async function installDeps (generator, deps, failed, resolutions, inputMap) {
173+
async function installDeps (generator, deps, failed, resolutions, inputMap, scopedResolutions) {
79174
const depNames = deps.map(([name]) => name);
80175
let needsRestart = false;
81176

@@ -90,12 +185,29 @@ async function installDeps (generator, deps, failed, resolutions, inputMap) {
90185
} catch (firstErr) {
91186
const errMsg = firstErr.message || String(firstErr);
92187
const failingPkg = extractFailingPackage(errMsg);
93-
if (failingPkg && !resolutions[failingPkg.name]) {
188+
const importingPkgScope = extractImportingPackageScope(errMsg);
189+
const hasGlobalPin = !!resolutions[failingPkg?.name];
190+
const hasPinnedImporterScope = !!(
191+
failingPkg &&
192+
importingPkgScope &&
193+
scopedResolutions[importingPkgScope]?.[failingPkg.name]
194+
);
195+
if (failingPkg && !hasGlobalPin && !hasPinnedImporterScope) {
94196
console.warn(`\x1b[33m [!] Import map: ${depSpec} failed — transitive dep ${failingPkg.name}@${failingPkg.version} not on CDN, searching for available version...\x1b[0m`);
95197
const goodVersion = await findAvailableCDNVersion(failingPkg.name, failingPkg.version);
96198
if (goodVersion) {
97-
console.log(`\x1b[32m [✓] Found ${failingPkg.name}@${goodVersion} on CDN, pinning via resolutions\x1b[0m`);
98-
resolutions[failingPkg.name] = goodVersion;
199+
const scopedPins = findScopedPinTargets(failingPkg.name, goodVersion, importingPkgScope)
200+
.map(toCachedScopeUrl)
201+
.filter(scopeUrl => scopedResolutions[scopeUrl]?.[failingPkg.name] !== goodVersion);
202+
if (scopedPins.length) {
203+
for (const scopeUrl of scopedPins) {
204+
(scopedResolutions[scopeUrl] ||= {})[failingPkg.name] = goodVersion;
205+
}
206+
console.log(`\x1b[32m [✓] Found ${failingPkg.name}@${goodVersion} on CDN, pinning via scoped locks for ${scopedPins.length} requester(s)\x1b[0m`);
207+
} else {
208+
console.log(`\x1b[32m [✓] Found ${failingPkg.name}@${goodVersion} on CDN, pinning via global resolutions\x1b[0m`);
209+
resolutions[failingPkg.name] = goodVersion;
210+
}
99211
needsRestart = true;
100212
continue; // don't mark as failed — will be retried in second pass
101213
} else {
@@ -119,8 +231,9 @@ async function installDeps (generator, deps, failed, resolutions, inputMap) {
119231
// If we discovered new resolution pins, recreate the generator and
120232
// redo the entire install so all deps benefit from the pins.
121233
if (needsRestart) {
122-
console.log(`\x1b[36m [↻] Restarting import map resolution with ${Object.keys(resolutions).length} pinned resolution(s)...\x1b[0m`);
123-
generator = createGenerator(inputMap, resolutions);
234+
const scopedPinCount = Object.values(scopedResolutions).reduce((sum, pins) => sum + Object.keys(pins || {}).length, 0);
235+
console.log(`\x1b[36m [↻] Restarting import map resolution with ${Object.keys(resolutions).length} global and ${scopedPinCount} scoped pinned resolution(s)...\x1b[0m`);
236+
generator = createGenerator(inputMap, resolutions, scopedResolutions);
124237
for (const key of Object.keys(failed)) delete failed[key];
125238
for (let dep of deps) {
126239
if (dep[0] == 'tar-fs' || isUnresolvableOnCDN(dep) || !!generator.map.imports[dep[0]]) continue;
@@ -154,18 +267,21 @@ export async function generateImportMap (packageName) {
154267
inputMap = JSON.parse((await cachedImportMap.read()).replace(/esm:\/\//g, 'https://')); // replace esm to make generator install again
155268
}
156269
const resolutions = {};
157-
let generator = createGenerator(inputMap, resolutions);
270+
const scopedResolutions = Object.assign({}, inputMap?._scopedResolutions || {});
271+
let generator = createGenerator(inputMap, resolutions, scopedResolutions);
158272
const failed = inputMap?._failed || {};
159273
generator = await installDeps(
160274
generator,
161275
Object.entries(pkg.config.dependencies || {}).filter(([dep]) => !dep.match(/lively(\.|-)/)),
162276
failed,
163277
resolutions,
164-
inputMap
278+
inputMap,
279+
scopedResolutions
165280
);
166281
const importMap = JSON.parse(JSON.stringify(generator.getMap()).replace(/https:\/\//g, 'esm://'))
167282
if (!obj.isEmpty(failed)) importMap._failed = failed;
168283
if (!obj.isEmpty(resolutions)) importMap._resolutions = resolutions;
284+
if (!obj.isEmpty(scopedResolutions)) importMap._scopedResolutions = scopedResolutions;
169285
if (!obj.isEmpty(importMap)) await cachedImportMap.writeJson(importMap);
170286
else if (inputMap) { await cachedImportMap.remove() }
171287
return importMap;

0 commit comments

Comments
 (0)