Skip to content

Commit 2ad4303

Browse files
catsoutcatsout
committed
node: Support yarn berry
1 parent 3fc0620 commit 2ad4303

5 files changed

Lines changed: 516 additions & 37 deletions

File tree

‎node/flatpak_node_generator/flatpak-yarn.js‎

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
const PackageManager = {
2+
Yarn1: `Yarn Classic`,
3+
Yarn2: `Yarn`,
4+
Npm: `npm`,
5+
Pnpm: `pnpm`,
6+
}
7+
8+
module.exports = {
9+
name: `flatpak-builder`,
10+
factory: require => {
11+
const { BaseCommand } = require(`@yarnpkg/cli`);
12+
const { parseSyml } = require('@yarnpkg/parsers');
13+
const { Configuration, Manifest, scriptUtils, structUtils, tgzUtils, execUtils, miscUtils, hashUtils } = require('@yarnpkg/core')
14+
const { Filename, ZipFS, npath, ppath, PortablePath, xfs } = require('@yarnpkg/fslib');
15+
const { getLibzipPromise } = require('@yarnpkg/libzip');
16+
const { gitUtils } = require('@yarnpkg/plugin-git');
17+
const { PassThrough, Readable, Writable } = require('stream');
18+
const { Command, Option } = require(`clipanion`);
19+
const { YarnVersion } = require('@yarnpkg/core');
20+
const fs = require('fs');
21+
22+
// from https://github.com/yarnpkg/berry/blob/%40yarnpkg/shell/3.2.3/packages/plugin-essentials/sources/commands/set/version.ts#L194
23+
async function setPackageManager(projectCwd) {
24+
const bundleVersion = YarnVersion;
25+
26+
const manifest = (await Manifest.tryFind(projectCwd)) || new Manifest();
27+
28+
if (bundleVersion && miscUtils.isTaggedYarnVersion(bundleVersion)) {
29+
manifest.packageManager = `yarn@${bundleVersion}`;
30+
const data = {};
31+
manifest.exportTo(data);
32+
33+
const path = ppath.join(projectCwd, Manifest.fileName);
34+
const content = `${JSON.stringify(data, null, manifest.indent)}\n`;
35+
36+
await xfs.changeFilePromise(path, content, {
37+
automaticNewlines: true,
38+
});
39+
}
40+
}
41+
42+
// func from https://github.com/yarnpkg/berry/blob/%40yarnpkg/shell/3.2.3/packages/yarnpkg-core/sources/scriptUtils.ts#L215
43+
async function prepareExternalProject(cwd, outputPath, { configuration, locator, stdout, yarn_v1, workspace = null }) {
44+
const devirtualizedLocator = locator && structUtils.isVirtualLocator(locator)
45+
? structUtils.devirtualizeLocator(locator)
46+
: locator;
47+
48+
const name = devirtualizedLocator
49+
? structUtils.stringifyLocator(devirtualizedLocator)
50+
: `an external project`;
51+
52+
const stderr = stdout;
53+
54+
stdout.write(`Packing ${name} from sources\n`);
55+
56+
const packageManagerSelection = await scriptUtils.detectPackageManager(cwd);
57+
let effectivePackageManager;
58+
if (packageManagerSelection !== null) {
59+
stdout.write(`Using ${packageManagerSelection.packageManager} for bootstrap. Reason: ${packageManagerSelection.reason}\n\n`);
60+
effectivePackageManager = packageManagerSelection.packageManager;
61+
} else {
62+
stdout.write(`No package manager configuration detected; defaulting to Yarn\n\n`);
63+
effectivePackageManager = PackageManager.Yarn2;
64+
}
65+
if (effectivePackageManager === PackageManager.Pnpm) {
66+
effectivePackageManager = PackageManager.Npm;
67+
}
68+
69+
const workflows = new Map([
70+
[PackageManager.Yarn1, async () => {
71+
const workspaceCli = workspace !== null
72+
? [`workspace`, workspace]
73+
: [];
74+
75+
await setPackageManager(cwd);
76+
77+
await Configuration.updateConfiguration(cwd, {
78+
yarnPath: yarn_v1,
79+
});
80+
81+
await xfs.appendFilePromise(ppath.join(cwd, `.npmignore`), `/.yarn\n`);
82+
83+
const pack = await execUtils.pipevp(`yarn`, [...workspaceCli, `pack`, `--filename`, npath.fromPortablePath(outputPath)], { cwd, stdout, stderr });
84+
if (pack.code !== 0)
85+
return pack.code;
86+
87+
return 0;
88+
}],
89+
[PackageManager.Yarn2, async () => {
90+
const workspaceCli = workspace !== null
91+
? [`workspace`, workspace]
92+
: [];
93+
const lockfilePath = ppath.join(cwd, Filename.lockfile);
94+
if (!(await xfs.existsPromise(lockfilePath)))
95+
await xfs.writeFilePromise(lockfilePath, ``);
96+
97+
const pack = await execUtils.pipevp(`yarn`, [...workspaceCli, `pack`, `--filename`, npath.fromPortablePath(outputPath)], { cwd, stdout, stderr });
98+
if (pack.code !== 0)
99+
return pack.code;
100+
return 0;
101+
}],
102+
[PackageManager.Npm, async () => {
103+
const workspaceCli = workspace !== null
104+
? [`--workspace`, workspace]
105+
: [];
106+
const packStream = new PassThrough();
107+
const packPromise = miscUtils.bufferStream(packStream);
108+
const pack = await execUtils.pipevp(`npm`, [`pack`, `--silent`, ...workspaceCli], { cwd, stdout: packStream, stderr });
109+
if (pack.code !== 0)
110+
return pack.code;
111+
112+
const packOutput = (await packPromise).toString().trim().replace(/^.*\n/s, ``);
113+
const packTarget = ppath.resolve(cwd, npath.toPortablePath(packOutput));
114+
await xfs.renamePromise(packTarget, outputPath);
115+
return 0;
116+
}],
117+
]);
118+
const workflow = workflows.get(effectivePackageManager);
119+
const code = await workflow();
120+
if (code === 0 || typeof code === `undefined`)
121+
return;
122+
else
123+
throw `Packing the package failed (exit code ${code})`;
124+
}
125+
126+
class convertToZipCommand extends BaseCommand {
127+
static paths = [[`convertToZip`]];
128+
yarn_v1 = Option.String({ required: true });
129+
130+
async execute() {
131+
const configuration = await Configuration.find(this.context.cwd,
132+
this.context.plugins);
133+
const lockfilePath = ppath.join(this.context.cwd, 'yarn.lock');
134+
const cacheFolder = `${configuration.get('globalFolder')}/cache`;
135+
const locatorFolder = `${cacheFolder}/locator`;
136+
137+
const compressionLevel = configuration.get(`compressionLevel`);
138+
const stdout = this.context.stdout;
139+
const gitChecksumPatches = []; // {name:, oriHash:, newHash:}
140+
141+
async function patchLockfileChecksum(lockfilePath, patches) {
142+
let currentContent = ``;
143+
try {
144+
currentContent = await xfs.readFilePromise(lockfilePath, `utf8`);
145+
} catch (error) {
146+
}
147+
const newContent = patches.reduce((acc, item, i) => {
148+
stdout.write(`patch '${item.name}' checksum:\n-${item.oriHash}\n+${item.newHash}\n\n\n`);
149+
const regex = new RegExp(item.oriHash, "g");
150+
return acc.replace(regex, item.newHash);
151+
}, currentContent);
152+
153+
await xfs.writeFilePromise(lockfilePath, newContent);
154+
}
155+
156+
async function getLockFileMeta(lockfilePath) {
157+
const content = await xfs.readFilePromise(lockfilePath, `utf8`);
158+
const parsed = parseSyml(content);
159+
return parsed.__metadata;
160+
}
161+
162+
const lockMeta = await getLockFileMeta(lockfilePath);
163+
stdout.write(`yarn lock: ${lockfilePath}\n`);
164+
stdout.write(`yarn lock version: ${lockMeta.version}\n`);
165+
stdout.write(`yarn lock cacheKey: ${lockMeta.cacheKey}\n`);
166+
167+
const convertToZip = async (tgz, target, opts) => {
168+
const tgzBuf = await xfs.readFilePromise(tgz);
169+
const fs = await tgzUtils.convertToZip(tgzBuf, opts);
170+
fs.discardAndClose();
171+
await xfs.copyFilePromise(fs.path, target);
172+
await xfs.unlinkPromise(fs.path);
173+
}
174+
175+
stdout.write(`converting tgz to zip: ${cacheFolder}\n`);
176+
177+
const files = fs.readdirSync(locatorFolder);
178+
const tasks = []
179+
for (const i in files) {
180+
const file = `${files[i]}`;
181+
let tgzFile = `${locatorFolder}/${file}`;
182+
const match = file.match(/([^-]+)-([^.]{1,10})[.](tgz|git)$/);
183+
if (!match) {
184+
stdout.write(`ignore ${file}\n`);
185+
continue;
186+
}
187+
let resolution, locator;
188+
const entry_type = match[3];
189+
const sha = match[2];
190+
let checksum;
191+
192+
if (entry_type === 'tgz') {
193+
resolution = Buffer.from(match[1], 'base64').toString();
194+
locator = structUtils.parseLocator(resolution, true);
195+
}
196+
else if (entry_type === 'git') {
197+
const gitJson = JSON.parse(fs.readFileSync(tgzFile, 'utf8'));
198+
199+
resolution = gitJson.resolution;
200+
locator = structUtils.parseLocator(resolution, true);
201+
checksum = gitJson.checksum;
202+
203+
const repoPathRel = gitJson.repo_dir_rel;
204+
205+
const cloneTarget = `${cacheFolder}/${repoPathRel}`;
206+
207+
const repoUrlParts = gitUtils.splitRepoUrl(locator.reference);
208+
const packagePath = ppath.join(cloneTarget, `package.tgz`);
209+
210+
await prepareExternalProject(cloneTarget, packagePath, {
211+
configuration: configuration,
212+
stdout,
213+
workspace: repoUrlParts.extra.workspace,
214+
locator,
215+
yarn_v1: this.yarn_v1,
216+
});
217+
218+
tgzFile = packagePath;
219+
220+
}
221+
const filename =
222+
`${structUtils.slugifyLocator(locator)}-${lockMeta.cacheKey}.zip`;
223+
const targetFile = `${cacheFolder}/${filename}`
224+
225+
tasks.push(async () => {
226+
await convertToZip(tgzFile, targetFile, {
227+
compressionLevel: compressionLevel,
228+
prefixPath: `node_modules/${structUtils.stringifyIdent(locator)}`,
229+
stripComponents: 1,
230+
});
231+
232+
if (entry_type === 'git') {
233+
const file_checksum = await hashUtils.checksumFile(targetFile);
234+
235+
if (file_checksum !== checksum) {
236+
const newSha = file_checksum.slice(0, 10);
237+
const newTarget = `${cacheFolder}/${structUtils.slugifyLocator(locator)}-${lockMeta.cacheKey}.zip`;
238+
fs.renameSync(targetFile, newTarget);
239+
240+
gitChecksumPatches.push({
241+
name: locator.name,
242+
oriHash: checksum,
243+
newHash: file_checksum,
244+
});
245+
}
246+
}
247+
});
248+
}
249+
250+
await Promise.all(tasks.map(t => t()));
251+
252+
patchLockfileChecksum(lockfilePath, gitChecksumPatches);
253+
stdout.write(`converting finished\n`);
254+
}
255+
}
256+
return {
257+
commands: [
258+
convertToZipCommand
259+
],
260+
};
261+
}
262+
};

‎node/flatpak_node_generator/manifest.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,9 +168,18 @@ def add_data_source(self, data: Union[str, bytes], destination: Path) -> None:
168168
self._add_source_with_destination(source, destination, is_dir=False)
169169

170170
def add_git_source(
171-
self, url: str, commit: str, destination: Optional[Path] = None
171+
self,
172+
url: str,
173+
commit: Optional[str] = None,
174+
destination: Optional[Path] = None,
175+
tag: Optional[str] = None,
172176
) -> None:
173-
source = {'type': 'git', 'url': url, 'commit': commit}
177+
source = {'type': 'git', 'url': url}
178+
assert commit or tag
179+
if commit:
180+
source['commit'] = commit
181+
if tag:
182+
source['tag'] = tag
174183
self._add_source_with_destination(source, destination, is_dir=True)
175184

176185
def add_script_source(self, commands: List[str], destination: Path) -> None:

‎node/flatpak_node_generator/providers/npm.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -308,14 +308,14 @@ async def resolve_source(self, package: Package) -> ResolvedSource:
308308
index = await self.registry_packages[package.name]
309309

310310
versions = index.data['versions']
311-
assert (
312-
package.version in versions
313-
), f'{package.name} versions available are {", ".join(versions)}, not {package.version}'
311+
assert package.version in versions, (
312+
f'{package.name} versions available are {", ".join(versions)}, not {package.version}'
313+
)
314314

315315
dist = versions[package.version]['dist']
316-
assert (
317-
'tarball' in dist
318-
), f'{package.name}@{package.version} has no tarball in dist'
316+
assert 'tarball' in dist, (
317+
f'{package.name}@{package.version} has no tarball in dist'
318+
)
319319

320320
index.used_versions.add(package.version)
321321

‎node/flatpak_node_generator/providers/special.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -155,9 +155,9 @@ async def _get_chromedriver_binary_version(self, package: Package) -> str:
155155
js = await Requests.instance.read_all(url, cachable=True)
156156
# XXX: a tad ugly
157157
match = re.search(r"exports\.version = '([^']+)'", js.decode())
158-
assert (
159-
match is not None
160-
), f'Failed to get ChromeDriver binary version from {url}'
158+
assert match is not None, (
159+
f'Failed to get ChromeDriver binary version from {url}'
160+
)
161161
return match.group(1)
162162

163163
async def _handle_electron_chromedriver(self, package: Package) -> None:
@@ -349,7 +349,6 @@ async def _handle_playwright(self, package: Package) -> None:
349349
url_tp = 'https://playwright.azureedge.net/builds/chromium/%d/%s'
350350
dl_file = 'chromium-linux.zip'
351351
elif name == 'chromium-headless-shell':
352-
# Shouldn't need the old scheme (didn't exist in 'browsers')
353352
url_tp = 'https://playwright.azureedge.net/builds/chromium/%d/%s'
354353
dl_file = 'chromium-headless-shell-linux.zip'
355354
elif name == 'firefox':

0 commit comments

Comments
 (0)